From 4aff840d761af61979d50c974fe2d4d9d20dff60 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:40:36 -0500 Subject: [PATCH 01/31] Keep the rest of extraParams.hxml when the release stamps the version Release stamp wrote extraParams.hxml from scratch with the two version defines, so the release package lost its third line, the macro that defines HXCPP_GC_GENERATIONAL and HXCPP_GC_INCREMENTAL_BARRIERS for -D HXCPP_GC_INCREMENTAL. With libhxcpp installed from a release, an incremental build stopped at hxcpp.h's "Incremental GC requires regenerated C++" error; from a checkout it built. The release workflow stamps before it packages, so 6.0.0-rc.2's package had the same file. Stamp now replaces only the libhxcpp and hxcpp define lines and keeps every other line as it is. test/release_stamp/check.sh stamps a copy and checks the defines and that each other line of the checkout's file survives; it runs in the haxelib job. --- .github/workflows/main.yml | 4 +++ Changes.md | 2 ++ test/release_stamp/check.sh | 55 +++++++++++++++++++++++++++++++++++++ tools/release/Release.hx | 14 ++++++++-- 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100755 test/release_stamp/check.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e69a4908a..6f897d715 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -324,6 +324,10 @@ jobs: # Git Bash on Windows shell: bash + - run: bash test/release_stamp/check.sh + name: What the release stamps into the package + shell: bash + # The test suites on every platform with the stable Haxe; nightly.yml runs # them with the Haxe nightly build once a day diff --git a/Changes.md b/Changes.md index 4bd812ea3..1d2f4e5d8 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,8 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now + libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/test/release_stamp/check.sh b/test/release_stamp/check.sh new file mode 100755 index 000000000..490b71c1d --- /dev/null +++ b/test/release_stamp/check.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Checks what `Release stamp` writes, in a copy of the files it reads and writes. +# Usage, from anywhere: bash test/release_stamp/check.sh +set -u + +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +failures=0 + +check() { + if [ "$2" = "$3" ]; then + echo "ok $1" + else + echo "FAIL $1" + echo " got: $2" + echo " expected: $3" + failures=$((failures + 1)) + fi +} + +# A release commit of 6.0.1: the version in haxelib.json and the top heading of Changes.md +sed 's/"version": "[^"]*"/"version": "6.0.1"/' "$ROOT/haxelib.json" > "$WORK/haxelib.json" +printf 'libhxcpp 6.0.1\n------------------------------------------------------------\n\n* A change\n' > "$WORK/Changes.md" +cp "$ROOT/extraParams.hxml" "$WORK/" +mkdir -p "$WORK/include" + +out=$(cd "$WORK" && haxe -p "$ROOT/tools/release" --run Release stamp 2>&1) +check "stamp succeeds" "$?" "0" +check "stamp reports the version" "$(printf '%s\n' "$out" | tr -d '\r' | head -1)" "version=6.0.1" + +stamped=$(tr -d '\r' < "$WORK/extraParams.hxml") +for name in libhxcpp hxcpp; do + check "extraParams defines $name as the version" \ + "$(printf '%s\n' "$stamped" | grep -c "^--macro haxe.macro.Compiler.define(\"$name\",\"6.0.1\")\$")" "1" +done +check "extraParams keeps no other version" "$(printf '%s\n' "$stamped" | grep -c 'Compiler.define("\(lib\)\?hxcpp","6.0.0")')" "0" + +# Every other line of the checkout's extraParams.hxml, such as the incremental +# collector's configuration macro, reaches the release as it is +while IFS= read -r line; do + case "$line" in + ""|*'Compiler.define("libhxcpp",'*|*'Compiler.define("hxcpp",'*) continue ;; + esac + check "extraParams keeps: $line" "$(printf '%s\n' "$stamped" | grep -cxF -- "$line")" "1" +done < <(tr -d '\r' < "$ROOT/extraParams.hxml") + +check "HxcppVersion.h names the version" "$(grep -c '"6.0.1"' "$WORK/include/HxcppVersion.h")" "2" + +# Stamping twice changes nothing more +cp "$WORK/extraParams.hxml" "$WORK/first.hxml" +(cd "$WORK" && haxe -p "$ROOT/tools/release" --run Release stamp > /dev/null 2>&1) +check "a second stamp leaves extraParams as it was" "$(cmp -s "$WORK/first.hxml" "$WORK/extraParams.hxml" && echo same)" "same" + +exit $failures diff --git a/tools/release/Release.hx b/tools/release/Release.hx index 94c96de66..c48c85125 100644 --- a/tools/release/Release.hx +++ b/tools/release/Release.hx @@ -98,8 +98,18 @@ class Release // that checks for hxcpp works the same whether it asked for hxcpp or libhxcpp. // A macro rather than -D lines: tools such as hxp take the line before a // library's -D line in `haxelib path` output as its path. - File.saveContent("extraParams.hxml", [for (name in ["libhxcpp", "hxcpp"]) - '--macro haxe.macro.Compiler.define("$name","$version")\n'].join("")); + // Only those two lines are rewritten: the file's other lines, such as the + // incremental collector's configuration macro, ship as they are. + File.saveContent("extraParams.hxml", stampedExtraParams(File.getContent("extraParams.hxml"), version)); + } + + static final DEFINE_LINE = ~/^--macro haxe\.macro\.Compiler\.define\("(libhxcpp|hxcpp)",/; + + static function stampedExtraParams(current:String, version:String):String + { + final defines = [for (name in ["libhxcpp", "hxcpp"]) '--macro haxe.macro.Compiler.define("$name","$version")']; + final kept = [for (line in current.replace("\r", "").split("\n")) if (line.trim() != "" && !DEFINE_LINE.match(line.trim())) line]; + return defines.concat(kept).join("\n") + "\n"; } // The first section of Changes.md: a heading line underlined with dashes, From 99b09f7050994f62304e1f8363648a177b545315 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:44:46 -0500 Subject: [PATCH 02/31] Link again when a target's flags, libraries or library paths change A link was redone only when an object or a library file on disk was newer than the output, or the list of objects changed. A flag or library added or removed alone left the output as it was, and the build reported success. On Emscripten every HXCPP_LINK_ define is link-only, and none reaches the compile options, so building a program with -D HXCPP_LINK_NO_EMBIND and again without it kept the program without Embind; the same held for HXCPP_LINK_STACK_SIZE, the memory growth defines and HXCPP_LINK_NODERAWFS. With MSVC, a -map: flag added to a target produced no map. The hash kept beside the output now covers the linker's and the target's flags, the libraries and the library paths as well as the objects, so a change to any of them links again. Every program links once more after updating, as its hash changes. test/link_flags builds once, then with a define that adds a map flag, and checks the map was written. --- .github/workflows/test.yml | 9 +++++++++ Changes.md | 1 + test/link_flags/Build.xml | 14 ++++++++++++++ test/link_flags/main.cpp | 7 +++++++ tools/hxcpp/Linker.hx | 5 ++++- 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 test/link_flags/Build.xml create mode 100644 test/link_flags/main.cpp diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 991d4fa26..6e74c2c33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -215,6 +215,15 @@ jobs: haxelib run hxcpp Build.xml test "$(tr -d '\r' < out/kept.txt)" = old test "$(tr -d '\r' < out/replaced.txt)" = new + - name: build tool links again when a link flag changes + continue-on-error: ${{ inputs.warn_only }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + working-directory: test/link_flags + run: | + haxelib run hxcpp Build.xml + rm -f prog.map + haxelib run hxcpp Build.xml -DWANT_MAP + test -f prog.map - name: build tool rejects file names that differ only in case continue-on-error: ${{ inputs.warn_only }} if: ${{ !cancelled() && steps.setup.outcome == 'success' }} diff --git a/Changes.md b/Changes.md index 1d2f4e5d8..a0f10f715 100644 --- a/Changes.md +++ b/Changes.md @@ -3,6 +3,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now +* Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/test/link_flags/Build.xml b/test/link_flags/Build.xml new file mode 100644 index 000000000..b2133d274 --- /dev/null +++ b/test/link_flags/Build.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + diff --git a/test/link_flags/main.cpp b/test/link_flags/main.cpp new file mode 100644 index 000000000..8135f6f32 --- /dev/null +++ b/test/link_flags/main.cpp @@ -0,0 +1,7 @@ +#include + +int main() +{ + printf("linked\n"); + return 0; +} diff --git a/tools/hxcpp/Linker.hx b/tools/hxcpp/Linker.hx index 9f49ca7be..391fd229e 100644 --- a/tools/hxcpp/Linker.hx +++ b/tools/hxcpp/Linker.hx @@ -130,7 +130,10 @@ class Linker var v18Added = false; var isOutOfDateLibs = false; - var md5 = Md5.encode(inObjs.join(";")); + // The objects, and what the command does with them: a flag or library + // added or removed relinks, as a changed object does + var md5 = Md5.encode(inObjs.join(";") + "|" + mFlags.concat(inTarget.mFlags).join(" ") + "|" + + libs.join(";") + "|" + inTarget.mLibPaths.join(";")); if (!FileSystem.exists(hashFile) || sys.io.File.getContent(hashFile)!=md5) isOutOfDateLibs = true; From 6dc7a9b2d16fea6591ef74e8a9b9c63ddf0916ac Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:50:44 -0500 Subject: [PATCH 03/31] Recompile when HXCPP_LINK_NO_EXCEPTIONS changes The build tool leaves every HXCPP_LINK_ define out of the options it compares between builds, taking them all for link options. On Emscripten HXCPP_LINK_NO_EXCEPTIONS also decides whether objects are compiled with -fwasm-exceptions. Adding it to a project built before without it recompiled nothing, and the link, now without -fwasm-exceptions, failed on __wasm_lpad_context and _Unwind_CallPersonality from the old objects (before the previous commit it did not link at all, so the program kept its exceptions). With the compile cache it worked, as the cache keys on the arguments. That one define is now compared like the others, so a change recompiles the groups that read Options.txt. test/emscripten/link_no_exceptions.sh builds without the cache, then again with the define, and runs it. --- .github/workflows/main.yml | 4 ++++ Changes.md | 1 + test/emscripten/link_no_exceptions.sh | 28 +++++++++++++++++++++++++++ tools/hxcpp/FileGroup.hx | 3 ++- 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 test/emscripten/link_no_exceptions.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f897d715..b18bb090b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -265,6 +265,10 @@ jobs: if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: timeout 1200 sh test/emscripten/exceptions/run.sh + - name: Rebuild with HXCPP_LINK_NO_EXCEPTIONS, without the compile cache + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: timeout 1200 sh test/emscripten/link_no_exceptions.sh + - name: Run the Embind test if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: timeout 600 sh test/emscripten/embind/run.sh diff --git a/Changes.md b/Changes.md index a0f10f715..1a2d45cd5 100644 --- a/Changes.md +++ b/Changes.md @@ -4,6 +4,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating +* Fixed an Emscripten build failing to link (undefined symbol: __wasm_lpad_context) when -D HXCPP_LINK_NO_EXCEPTIONS was added to a project built before without it: the define also compiles without -fwasm-exceptions, but the build tool took every HXCPP_LINK_ define for a link option and compiled nothing again. It recompiles now, and once back without it libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/test/emscripten/link_no_exceptions.sh b/test/emscripten/link_no_exceptions.sh new file mode 100644 index 000000000..c96b7bfaa --- /dev/null +++ b/test/emscripten/link_no_exceptions.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Builds Main with exceptions, then again into the same folder with +# HXCPP_LINK_NO_EXCEPTIONS, which compiles without -fwasm-exceptions as well as +# linking without it, and runs that build under node. Without the compile +# cache, so the second build has only the stored options to tell it that +# every object must be compiled again. Needs what run.sh does. +# Usage: test/emscripten/link_no_exceptions.sh [extra haxe args] + +cd "$(dirname "$0")" +unset HXCPP_COMPILE_CACHE +dir=bin/toggle-exceptions +haxe -m Main -D emscripten --cpp $dir "$@" || { echo "Failed to build with exceptions"; exit 1; } +haxe -m Main -D emscripten --cpp $dir -D HXCPP_LINK_NO_EXCEPTIONS "$@" || { echo "Failed to build again with HXCPP_LINK_NO_EXCEPTIONS"; exit 1; } +if ! actual=$(node $dir/Main.js); then + echo "Failed: return code was $?" + exit 1 +fi +expected=$(tr -d '\r' < stdout.txt) +actual=$(printf '%s' "$actual" | tr -d '\r') +if [ "$actual" != "$expected" ]; then + echo "Failed: output did not match" + echo "Expected stdout:" + echo "$expected" + echo "Actual stdout:" + echo "$actual" + exit 1 +fi +echo "Emscripten HXCPP_LINK_NO_EXCEPTIONS rebuild test passed" diff --git a/tools/hxcpp/FileGroup.hx b/tools/hxcpp/FileGroup.hx index 71248e1b4..2d0a14aae 100644 --- a/tools/hxcpp/FileGroup.hx +++ b/tools/hxcpp/FileGroup.hx @@ -226,7 +226,8 @@ class FileGroup { // Haxe writes defines as they were given, so -D HXCPP_M64 arrives upper case var name = def.split("=")[0].toLowerCase(); - if (name.indexOf("hxcpp_link")>=0) + // Emscripten compiles with -fwasm-exceptions unless HXCPP_LINK_NO_EXCEPTIONS + if (name.indexOf("hxcpp_link")>=0 && name!="hxcpp_link_no_exceptions") { // Only effects linking, not compiling } From eb6a190b7995fd05a14e3de9d04b4ea1d99e5b02 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:51:05 -0500 Subject: [PATCH 04/31] Read the build tool's sources from its own folder in the defines command `haxelib run hxcpp defines` lists the defines the build tool reads by scanning tools/hxcpp/*.hx, but opened each file by its bare name, which resolves against the caller's folder, as haxelib runs the tool there. From anywhere but tools/hxcpp it failed with "Could not read .../tools/hxcpp : [file_contents,BuildTool.hx]" and exit code 1. It opens them in that folder now. The build tool tests run it and look for HXCPP_COMPILE_CACHE in the list. --- .github/workflows/test.yml | 7 +++++++ Changes.md | 1 + tools/hxcpp/BuildTool.hx | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e74c2c33..98547ba7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -224,6 +224,13 @@ jobs: rm -f prog.map haxelib run hxcpp Build.xml -DWANT_MAP test -f prog.map + - name: build tool lists the defines it reads + continue-on-error: ${{ inputs.warn_only }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + working-directory: test/link_flags + run: | + haxelib run hxcpp defines > defines.txt + grep -qx HXCPP_COMPILE_CACHE defines.txt - name: build tool rejects file names that differ only in case continue-on-error: ${{ inputs.warn_only }} if: ${{ !cancelled() && steps.setup.outcome == 'success' }} diff --git a/Changes.md b/Changes.md index 1a2d45cd5..909b4de29 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating * Fixed an Emscripten build failing to link (undefined symbol: __wasm_lpad_context) when -D HXCPP_LINK_NO_EXCEPTIONS was added to a project built before without it: the define also compiles without -fwasm-exceptions, but the build tool took every HXCPP_LINK_ define for a link option and compiled nothing again. It recompiles now, and once back without it +* Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/tools/hxcpp/BuildTool.hx b/tools/hxcpp/BuildTool.hx index 7fa8dc4d3..df8da7d87 100644 --- a/tools/hxcpp/BuildTool.hx +++ b/tools/hxcpp/BuildTool.hx @@ -1867,7 +1867,7 @@ class BuildTool var allDefines = new Map(); for(file in FileSystem.readDirectory(dir)) if (file.endsWith(".hx")) - for(line in sys.io.File.getContent(file).split("\n")) + for(line in sys.io.File.getContent('$dir/$file').split("\n")) if (defineMatch.match(line)) allDefines.set(defineMatch.matched(1),true); for(key in allDefines.keys()) From 1f5f6e74781371d435a0810af672eb9a017f9315 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:52:49 -0500 Subject: [PATCH 05/31] Run the ARM64 vcvars script for HXCPP_MSVC_VER on an ARM64 build With HXCPP_MSVC_VER, msvc-arm64-setup.bat sets up the install it names through that install's vcvars_arm64.bat, but it spelled the x64 host as msvc_host_arch has it, x64, where Visual Studio's script is vcvarsamd64_arm64.bat, so on an x64 machine it stopped with "does not have vcvarsx64_arm64.bat". On an ARM64 machine (host x86) it checked for vcvarsx86_arm64.bat and then called vcvarsx86_amd64.bat, setting up the x64 compiler for an ARM64 build. It now names the x64 host amd64 and calls the _arm64 script it checked for. test/msvc_setup/check.ps1 runs the script against a stand-in install whose vcvars scripts say which ran, for both hosts. --- Changes.md | 1 + test/msvc_setup/check.ps1 | 23 +++++++++++++++++++++++ toolchain/msvc-arm64-setup.bat | 9 ++++++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Changes.md b/Changes.md index 909b4de29..79a967405 100644 --- a/Changes.md +++ b/Changes.md @@ -6,6 +6,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating * Fixed an Emscripten build failing to link (undefined symbol: __wasm_lpad_context) when -D HXCPP_LINK_NO_EXCEPTIONS was added to a project built before without it: the define also compiles without -fwasm-exceptions, but the build tool took every HXCPP_LINK_ define for a link option and compiled nothing again. It recompiles now, and once back without it * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again +* Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/test/msvc_setup/check.ps1 b/test/msvc_setup/check.ps1 index 4aef34acb..17f81e227 100644 --- a/test/msvc_setup/check.ps1 +++ b/test/msvc_setup/check.ps1 @@ -34,5 +34,28 @@ foreach ($script in "msvc-setup", "msvc64-setup", "msvc-arm64-setup", "msvc-winr } } ${env:ProgramFiles(x86)} = $realPf86 + +# HXCPP_MSVC_VER names an install by its Common7\Tools folder, and the ARM64 +# script runs that install's vcvars script for the host. A stand-in install +# whose vcvars scripts only say which one ran, named as Visual Studio names them +$fake = "$work\fake vs" +New-Item -ItemType Directory -Force "$fake\Common7\Tools", "$fake\VC\Auxiliary\Build" | Out-Null +foreach ($name in "x86_amd64", "x86_arm64", "amd64_x86", "amd64_arm64") { + Set-Content -Encoding ascii "$fake\VC\Auxiliary\Build\vcvars$name.bat" "@set VCVARS_RAN=$name" +} +$env:HXCPP_MSVC = "$fake\Common7\Tools\" +foreach ($case in @(@{ host = "x64"; expect = "amd64_arm64" }, @{ host = "x86"; expect = "x86_arm64" })) { + $env:msvc_host_arch = $case.host + $out = cmd /c "$root\toolchain\msvc-arm64-setup.bat" 2>&1 | Out-String + if ($out -match "(?m)^HXCPP_VARS" -and $out -match "(?m)^VCVARS_RAN=$($case.expect)\s*$") { + "ok msvc-arm64-setup with HXCPP_MSVC on an $($case.host) host" + } else { + "FAIL msvc-arm64-setup with HXCPP_MSVC on an $($case.host) host: expected vcvars$($case.expect).bat" + $out -split "`r?`n" | Where-Object { $_ -match "^(Error|VCVARS_RAN|HXCPP_VARS)" } + $failures++ + } +} +Remove-Item Env:HXCPP_MSVC + Remove-Item -Recurse -Force $work exit $failures diff --git a/toolchain/msvc-arm64-setup.bat b/toolchain/msvc-arm64-setup.bat index ec85813f8..b3ba42657 100644 --- a/toolchain/msvc-arm64-setup.bat +++ b/toolchain/msvc-arm64-setup.bat @@ -2,12 +2,15 @@ setlocal @set "vswhere=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" @set InstallDir= @if exist "%vswhere%" for /f "usebackq tokens=*" %%i in (`"%vswhere%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.arm64 -property installationPath`) do @set "InstallDir=%%i" +@rem Visual Studio names the x64 host amd64 in its vcvars scripts +@set "vcvars_host=%msvc_host_arch%" +@if /i "%vcvars_host%"=="x64" set "vcvars_host=amd64" @if exist "%HXCPP_MSVC%\..\..\VC\" ( - @if not exist "%hxcpp_msvc%\..\..\VC\Auxiliary\Build\vcvars%msvc_host_arch%_arm64.bat" ( - @echo Error: the specified MSVC version does not have vcvars%msvc_host_arch%_arm64.bat setup script + @if not exist "%hxcpp_msvc%\..\..\VC\Auxiliary\Build\vcvars%vcvars_host%_arm64.bat" ( + @echo Error: the specified MSVC version does not have vcvars%vcvars_host%_arm64.bat setup script ) else ( @echo "%HXCPP_MSVC%" - @call "%HXCPP_MSVC%\..\..\VC\Auxiliary\Build\vcvars%msvc_host_arch%_amd64.bat" + @call "%HXCPP_MSVC%\..\..\VC\Auxiliary\Build\vcvars%vcvars_host%_arm64.bat" @echo HXCPP_VARS @set ) From b7d6fde34fe1b506cb21bb9d8dc858b84517205a Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:56:00 -0500 Subject: [PATCH 06/31] Link again after a strip or manifest step that failed after the link The linker writes its record of the link, the .hash beside the output, as soon as the link succeeds, and the strip, manifest and PGO steps run after it. When one of those failed the build stopped with its error, but the output and its record were in place, so the next build found nothing to do and succeeded with the output unstripped or without its manifest: with -DmanifestFile set to a manifest mt.exe cannot parse, the second build exited 0. A failure after the link now deletes the record on the way out, so the next build links and runs the steps again. test/link_flags builds twice with a bad manifest and expects both to fail. --- .github/workflows/test.yml | 12 ++++++++++++ Changes.md | 1 + test/link_flags/bad.manifest | 1 + tools/hxcpp/BuildTool.hx | 9 +++++++++ tools/hxcpp/Linker.hx | 4 ++++ 5 files changed, 27 insertions(+) create mode 100644 test/link_flags/bad.manifest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98547ba7c..88d2cdb55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -224,6 +224,18 @@ jobs: rm -f prog.map haxelib run hxcpp Build.xml -DWANT_MAP test -f prog.map + - name: build tool fails again after a step after the link failed + continue-on-error: ${{ inputs.warn_only }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' && runner.os == 'Windows' }} + working-directory: test/link_flags + run: | + rm -rf obj prog.exe prog.exe.hash + for build in first second; do + if haxelib run hxcpp Build.xml -DmanifestFile=bad.manifest; then + echo "::error::the $build build with a bad manifest should have failed" + exit 1 + fi + done - name: build tool lists the defines it reads continue-on-error: ${{ inputs.warn_only }} if: ${{ !cancelled() && steps.setup.outcome == 'success' }} diff --git a/Changes.md b/Changes.md index 79a967405..d30306f03 100644 --- a/Changes.md +++ b/Changes.md @@ -7,6 +7,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed an Emscripten build failing to link (undefined symbol: __wasm_lpad_context) when -D HXCPP_LINK_NO_EXCEPTIONS was added to a project built before without it: the define also compiles without -fwasm-exceptions, but the build tool took every HXCPP_LINK_ define for a link option and compiled nothing again. It recompiles now, and once back without it * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host +* Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/test/link_flags/bad.manifest b/test/link_flags/bad.manifest new file mode 100644 index 000000000..d2ef466ec --- /dev/null +++ b/test/link_flags/bad.manifest @@ -0,0 +1 @@ +This is not a manifest, so mt.exe fails on it after the link. diff --git a/tools/hxcpp/BuildTool.hx b/tools/hxcpp/BuildTool.hx index df8da7d87..e0148c1d9 100644 --- a/tools/hxcpp/BuildTool.hx +++ b/tools/hxcpp/BuildTool.hx @@ -789,6 +789,14 @@ class BuildTool if (output!="") { + // The link is recorded as done, so a step after it that fails + // would leave the output looking up to date: forget the link + var unfinished = linker.mLastHashFile; + Tools.addOnExitHook(function(code:Int) { + if (code!=0 && unfinished!=null) + try { FileSystem.deleteFile(unfinished); } catch(e:Dynamic) { } + }); + if (target.mToolID=="exe" || target.mToolID=="dll") Pgo.linked(output); @@ -824,6 +832,7 @@ class BuildTool mManifester.add(output, manifest,target.mToolID=="exe"); } } + unfinished = null; } diff --git a/tools/hxcpp/Linker.hx b/tools/hxcpp/Linker.hx index 391fd229e..cb87c25e4 100644 --- a/tools/hxcpp/Linker.hx +++ b/tools/hxcpp/Linker.hx @@ -19,6 +19,8 @@ class Linker public var mExpandArchives:Bool; public var mRecreate:Bool; public var mLastOutName:String; + // The record of the last link this call made, or null when it linked nothing + public var mLastHashFile:String; public var mAddLibPath:String; public function new(inExe:String) @@ -103,6 +105,7 @@ class Linker } mLastOutName = out_name; + mLastHashFile = null; var lastLib = ""; var libs = new Array(); @@ -307,6 +310,7 @@ class Linker } sys.io.File.saveContent(hashFile,md5); + mLastHashFile = hashFile; return out_name; } From c3a9ee48e37ff85e1f9f3842abd38215c46115ff Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:45:23 -0500 Subject: [PATCH 07/31] Do nothing at a safe point inside a GC-free zone The fast paths of Mutex, Condition, Semaphore, Lock and Deque stop at a pending collection through __hxcpp_gc_safe_point (be8debe6, cf748b7e, bdf84d88). Mutex and Condition may be used inside a zone the caller has already entered, since their guard leaves only a zone it entered (603fd03c). There the stop paused the thread as if it were outside the zone: during a collection PauseForCollect ended the process with "Bad Allocation while collecting", and before one it signalled and waited for mCollectDone, which the collector only sends to a thread it found outside a zone. A thread in a zone is already safe, so the safe point now returns there. The allocation paths still call PauseForCollect directly, so allocating inside a zone keeps its diagnostic. Test: SafePointInGcFreeZone takes Mutex and Condition in a loop inside a zone while the main thread collects. It ended with the critical error in 3 of 3 runs before the change, and passes 20 of 20 after it, Windows x64, MSVC 19.44. --- Changes.md | 1 + src/hx/gc/Immix.cpp | 17 +++++- test/regression/SafePointInGcFreeZone/Main.hx | 61 +++++++++++++++++++ .../SafePointInGcFreeZone/build.hxml | 2 + .../SafePointInGcFreeZone/stdout.txt | 1 + test/regression/emscripten-skip.txt | 1 + 6 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 test/regression/SafePointInGcFreeZone/Main.hx create mode 100644 test/regression/SafePointInGcFreeZone/build.hxml create mode 100644 test/regression/SafePointInGcFreeZone/stdout.txt diff --git a/Changes.md b/Changes.md index d30306f03..436297848 100644 --- a/Changes.md +++ b/Changes.md @@ -8,6 +8,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host * Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again +* Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index 30b9a5334..28d3768ba 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -8184,6 +8184,7 @@ class LocalAllocator : public hx::StackContext void PopTopOfStack() { } void SetBottomOfStack(int *inBottom) { } void PauseForCollect() { } + void SafePoint() { } void EnterGCFreeZone() { } bool TryGCFreeZone() { return true; } bool TryExitGCFreeZone() { return false; } @@ -8283,6 +8284,17 @@ class LocalAllocator : public hx::StackContext #endif } + // A thread in a GC-free zone is already safe: the collector does not wait + // for it, and does not release it from PauseForCollect either + void SafePoint() + { + #ifndef HXCPP_SINGLE_THREADED_APP + if (mGCFreeZone) + return; + #endif + PauseForCollect(); + } + void EnterGCFreeZone() { #ifndef HXCPP_SINGLE_THREADED_APP @@ -9593,10 +9605,13 @@ void __hxcpp_exit_gc_free_zone() } +// The thread primitives' fast paths call this, and Mutex and Condition may be +// used from inside a zone: pausing there during a collection was "Bad +// Allocation while collecting", or waited for a release that never came void __hxcpp_gc_safe_point() { if (hx::gPauseForCollect) - hx::PauseForCollect(); + GetLocalAlloc()->SafePoint(); } //#define HXCPP_FORCE_OBJ_MAP diff --git a/test/regression/SafePointInGcFreeZone/Main.hx b/test/regression/SafePointInGcFreeZone/Main.hx new file mode 100644 index 000000000..812fd5371 --- /dev/null +++ b/test/regression/SafePointInGcFreeZone/Main.hx @@ -0,0 +1,61 @@ +import cpp.vm.Gc; +import sys.thread.Condition; +import sys.thread.Lock; +import sys.thread.Mutex; +import sys.thread.Thread; + +// Mutex and Condition take their locks inside a GC-free zone the caller already +// entered, and their uncontended fast paths stop at a pending collection. That +// stop paused the thread as if it were outside a zone: during a collection the +// process ended with "Bad Allocation while collecting", and otherwise the thread +// waited for a release the collector never sends to a thread in a zone. A +// thread in a zone is already safe, so the stop now does nothing there. +class Main { + static var count = 0; + static var stop = false; + + static function main() { + var finished = new Lock(); + var mutex = new Mutex(); + var condition = new Condition(); + Thread.create(() -> { + while (!stop) { + Gc.enterGCFreeZone(); + mutex.acquire(); + mutex.release(); + if (mutex.tryAcquire()) + mutex.release(); + condition.acquire(); + condition.release(); + if (condition.tryAcquire()) + condition.release(); + Gc.exitGCFreeZone(); + count++; + } + finished.release(); + }); + var stalled = false; + for (round in 0...200) { + var junk = []; + for (i in 0...2000) + junk.push([i]); + Gc.run(false); + var before = count; + Sys.sleep(0.005); + if (count == before) { + Sys.sleep(2.0); + if (count == before) { + stalled = true; + break; + } + } + } + stop = true; + if (stalled) { + Sys.println("the thread in the GC-free zone stopped"); + Sys.exit(1); + } + finished.wait(); + Sys.println("the thread in the GC-free zone ran through 200 collections"); + } +} diff --git a/test/regression/SafePointInGcFreeZone/build.hxml b/test/regression/SafePointInGcFreeZone/build.hxml new file mode 100644 index 000000000..f35632dbc --- /dev/null +++ b/test/regression/SafePointInGcFreeZone/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-m Main diff --git a/test/regression/SafePointInGcFreeZone/stdout.txt b/test/regression/SafePointInGcFreeZone/stdout.txt new file mode 100644 index 000000000..ce97c075a --- /dev/null +++ b/test/regression/SafePointInGcFreeZone/stdout.txt @@ -0,0 +1 @@ +the thread in the GC-free zone ran through 200 collections diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index 45d8d7506..64677e53c 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -91,6 +91,7 @@ RegexpNativeMemoryPressure: threads (std::thread, which needs a pthreads build) RegexpObsoleteSubject: threads (std::thread, which needs a pthreads build) RegisterHeldRoots: threads RootWalkRace: threads (std::thread, which needs a pthreads build) +SafePointInGcFreeZone: threads SemaphoreLoopCollect: threads SocketEmptyBytes: sockets SocketIpv6Accept: sockets From 4563191b352621ac869ac2cd13b1170dd75353f5 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:48:18 -0500 Subject: [PATCH 08/31] Keep the caller's GC-free zone in Lock, Semaphore and Deque waits The zone state is a flag, not a depth. Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and Deque's blocking pop and contended lock entered a zone with EnterGCFreeZone and left it with ExitGCFreeZone, so called inside a zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone around a blocking call, they took the thread out of it, and the caller's own exit ended the process with "GCFree Zone mismatch". Mutex and Condition already kept the caller's zone through hx::AutoGCFreeZone, which leaves only a zone it entered (603fd03c). They use the guard too, closing it before a throw. The POSIX and Apple semaphores and the POSIX Deque branch are not compiled on Windows; the change there is the same substitution. Test: GcFreeZoneThreadWaits waits on each inside a zone, with a second thread releasing the ones without a timeout. It ended with the critical error at the first wait before the change, and passes 10 of 10 after it, Windows x64, MSVC 19.44. --- Changes.md | 1 + src/hx/Thread.cpp | 18 ++++---- src/hx/thread/CountingSemaphore.apple.cpp | 7 ++- src/hx/thread/CountingSemaphore.posix.cpp | 12 +++-- src/hx/thread/CountingSemaphore.win32.cpp | 13 +++--- test/regression/GcFreeZoneThreadWaits/Main.hx | 45 +++++++++++++++++++ .../GcFreeZoneThreadWaits/build.hxml | 2 + .../GcFreeZoneThreadWaits/stdout.txt | 6 +++ test/regression/emscripten-skip.txt | 1 + 9 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 test/regression/GcFreeZoneThreadWaits/Main.hx create mode 100644 test/regression/GcFreeZoneThreadWaits/build.hxml create mode 100644 test/regression/GcFreeZoneThreadWaits/stdout.txt diff --git a/Changes.md b/Changes.md index 436297848..52acfb4e1 100644 --- a/Changes.md +++ b/Changes.md @@ -8,6 +8,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host * Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again +* Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there libhxcpp 6.0.0-rc.2 diff --git a/src/hx/Thread.cpp b/src/hx/Thread.cpp index b7f875746..6368bea72 100644 --- a/src/hx/Thread.cpp +++ b/src/hx/Thread.cpp @@ -131,9 +131,9 @@ struct Deque : public Array_obj __hxcpp_gc_safe_point(); if (!inMutex.TryLock()) { - hx::EnterGCFreeZone(); + // Keeps a zone the caller is in, as Mutex does + hx::AutoGCFreeZone zone; inMutex.Lock(); - hx::ExitGCFreeZone(); } } @@ -165,9 +165,10 @@ struct Deque : public Array_obj { mSemaphore.Reset(); mMutex.Unlock(); - hx::EnterGCFreeZone(); - mSemaphore.Wait(); - hx::ExitGCFreeZone(); + { + hx::AutoGCFreeZone zone; + mSemaphore.Wait(); + } lockFor(mMutex); } } @@ -204,10 +205,9 @@ struct Deque : public Array_obj lockFor(mSemaphore.mMutex); if (inBlock && !hasItems()) { - hx::EnterGCFreeZone(); + hx::AutoGCFreeZone zone; while(!hasItems()) mSemaphore.QWait(); - hx::ExitGCFreeZone(); } Dynamic result = popHead(); if (hasItems()) @@ -459,12 +459,12 @@ class hxLock : public hx::Object return false; } - hx::EnterGCFreeZone(); + // Keeps a zone the caller is in, as Mutex does + hx::AutoGCFreeZone zone; if (inTimeout<0) mNotEmpty.Wait( ); else mNotEmpty.WaitSeconds(wait); - hx::ExitGCFreeZone(); } } void Release() diff --git a/src/hx/thread/CountingSemaphore.apple.cpp b/src/hx/thread/CountingSemaphore.apple.cpp index a0d8100dd..25564db0e 100644 --- a/src/hx/thread/CountingSemaphore.apple.cpp +++ b/src/hx/thread/CountingSemaphore.apple.cpp @@ -37,15 +37,14 @@ hx::thread::CountingSemaphore_obj::CountingSemaphore_obj(int value) : impl(new I void hx::thread::CountingSemaphore_obj::acquire() { - hx::EnterGCFreeZone(); + // Keeps a zone the caller is in, as Mutex does + hx::AutoGCFreeZone zone; if (0 != dispatch_semaphore_wait(impl->semaphore, DISPATCH_TIME_FOREVER)) { - hx::ExitGCFreeZone(); + zone.close(); hx::Throw(HX_CSTRING("Failed to wait for semaphore")); } - - hx::ExitGCFreeZone(); } void hx::thread::CountingSemaphore_obj::release() diff --git a/src/hx/thread/CountingSemaphore.posix.cpp b/src/hx/thread/CountingSemaphore.posix.cpp index 9f30de720..8cbb3353d 100644 --- a/src/hx/thread/CountingSemaphore.posix.cpp +++ b/src/hx/thread/CountingSemaphore.posix.cpp @@ -46,7 +46,8 @@ void hx::thread::CountingSemaphore_obj::acquire() if (0 == sem_trywait(&impl->semaphore)) return; - hx::EnterGCFreeZone(); + // Keeps a zone the caller is in, as Mutex does + hx::AutoGCFreeZone zone; // sem_wait is never restarted after a caught signal, regardless of // SA_RESTART - retry on EINTR rather than throwing spuriously @@ -59,11 +60,9 @@ void hx::thread::CountingSemaphore_obj::acquire() if (0 != r) { - hx::ExitGCFreeZone(); + zone.close(); hx::Throw(HX_CSTRING("Failed to wait on semaphore")); } - - hx::ExitGCFreeZone(); } void hx::thread::CountingSemaphore_obj::release() @@ -93,7 +92,7 @@ bool hx::thread::CountingSemaphore_obj::tryAcquire(Null timeout) } else { - hx::EnterGCFreeZone(); + hx::AutoGCFreeZone zone; struct timespec t; #ifdef HX_SEM_CLOCKWAIT @@ -118,12 +117,11 @@ bool hx::thread::CountingSemaphore_obj::tryAcquire(Null timeout) if (0 == r) { - hx::ExitGCFreeZone(); return true; } else { - hx::ExitGCFreeZone(); + zone.close(); if (error == ETIMEDOUT) { return false; diff --git a/src/hx/thread/CountingSemaphore.win32.cpp b/src/hx/thread/CountingSemaphore.win32.cpp index 0ac01c80f..f9d9ab8e3 100644 --- a/src/hx/thread/CountingSemaphore.win32.cpp +++ b/src/hx/thread/CountingSemaphore.win32.cpp @@ -39,15 +39,14 @@ void hx::thread::CountingSemaphore_obj::acquire() if (NO_ERROR == WaitForSingleObject(impl->semaphore, 0)) return; - hx::EnterGCFreeZone(); + // Keeps a zone the caller is in, as Mutex does + hx::AutoGCFreeZone zone; if (NO_ERROR != WaitForSingleObject(impl->semaphore, INFINITE)) { - hx::ExitGCFreeZone(); + zone.close(); hx::Throw(HX_CSTRING("Failed to wait for semaphore")); } - - hx::ExitGCFreeZone(); } void hx::thread::CountingSemaphore_obj::release() @@ -85,18 +84,16 @@ bool hx::thread::CountingSemaphore_obj::tryAcquire(Null timeout) return hx::Throw(HX_CSTRING("Failed to wait for semaphore")); } - hx::EnterGCFreeZone(); + hx::AutoGCFreeZone zone; switch (WaitForSingleObject(impl->semaphore, waitMs)) { case NO_ERROR: - hx::ExitGCFreeZone(); return true; case WAIT_TIMEOUT: - hx::ExitGCFreeZone(); return false; default: - hx::ExitGCFreeZone(); + zone.close(); return hx::Throw(HX_CSTRING("Failed to wait for semaphore")); } } \ No newline at end of file diff --git a/test/regression/GcFreeZoneThreadWaits/Main.hx b/test/regression/GcFreeZoneThreadWaits/Main.hx new file mode 100644 index 000000000..ce25af982 --- /dev/null +++ b/test/regression/GcFreeZoneThreadWaits/Main.hx @@ -0,0 +1,45 @@ +import cpp.vm.Gc; +import sys.thread.Deque; +import sys.thread.Lock; +import sys.thread.Semaphore; +import sys.thread.Thread; + +// Lock, Semaphore and Deque left a GC-free zone their caller had entered when +// they blocked: they entered a zone and left it, and the thread's zone state is +// a flag, not a depth. The caller's own exit then ended the process with +// "GCFree Zone mismatch". Mutex and Condition already kept the caller's zone. +// Everything is allocated before the zone is entered. +function inZone(name:String, wait:() -> Void) { + Gc.enterGCFreeZone(); + wait(); + Gc.exitGCFreeZone(); + Sys.println(name + ": the caller's zone was kept"); +} + +function later(action:() -> Void) { + Thread.create(() -> { + Sys.sleep(0.05); + action(); + }); +} + +function main() { + var unreleased = new Lock(); + inZone("Lock.wait with a timeout", () -> unreleased.wait(0.01)); + var lock = new Lock(); + later(() -> lock.release()); + inZone("Lock.wait", () -> lock.wait()); + + var empty = new Semaphore(0); + inZone("Semaphore.tryAcquire with a timeout", () -> empty.tryAcquire(0.01)); + var semaphore = new Semaphore(0); + later(() -> semaphore.release()); + inZone("Semaphore.acquire", () -> semaphore.acquire()); + + var deque = new Deque(); + later(() -> deque.add(1)); + inZone("Deque.pop", () -> deque.pop(true)); + + Gc.run(true); + Sys.println("collection works afterwards"); +} diff --git a/test/regression/GcFreeZoneThreadWaits/build.hxml b/test/regression/GcFreeZoneThreadWaits/build.hxml new file mode 100644 index 000000000..f35632dbc --- /dev/null +++ b/test/regression/GcFreeZoneThreadWaits/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-m Main diff --git a/test/regression/GcFreeZoneThreadWaits/stdout.txt b/test/regression/GcFreeZoneThreadWaits/stdout.txt new file mode 100644 index 000000000..9fc36d95d --- /dev/null +++ b/test/regression/GcFreeZoneThreadWaits/stdout.txt @@ -0,0 +1,6 @@ +Lock.wait with a timeout: the caller's zone was kept +Lock.wait: the caller's zone was kept +Semaphore.tryAcquire with a timeout: the caller's zone was kept +Semaphore.acquire: the caller's zone was kept +Deque.pop: the caller's zone was kept +collection works afterwards diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index 64677e53c..a0d1016bf 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -34,6 +34,7 @@ FinalizersAcrossFreeZones: threads GcConcurrentCensus: threads (std::thread, which needs a pthreads build) GcFinalizerPromotion: threads (std::thread, which needs a pthreads build) GcFreeZoneNestedGuard: single-threaded Emscripten builds have no GC-free zones, so a guard enters none +GcFreeZoneThreadWaits: threads GcHeapGrows: expects the desktop 20 MB working target; other platforms, Emscripten included, use 8 MB GcIdentityBurst: threads (std::thread, which needs a pthreads build) GcLargeInteriorRoot: threads (std::thread, which needs a pthreads build) From d4c2b0d5972d87278187251fc715445a22758e68 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:54:05 -0500 Subject: [PATCH 09/31] Sleep in parts that fit the native call Sys.sleep converted the duration straight to the native call's type: on Windows to a DWORD of milliseconds, which from 2^32 ms, about 49.7 days, wrapped to a short sleep or none, and elsewhere to an int of seconds, which overflowed past 2^31 s. Infinity, a way to sleep for good, returned at once on both. Windows now sleeps in parts of 0xfffffffe ms, one short of INFINITE, and POSIX in parts of 1e9 s, so an infinite sleep never returns. The POSIX branch is not compiled on Windows. Test: SleepLongDuration checks that sleeps of 4294967.3 s, 3e9 s and infinity are still running after 0.3 s, and that short, zero, negative and NaN sleeps return. Before the change, 4294967.3 s and infinity returned at once on Windows x64; after it all pass, 5 of 5. --- Changes.md | 1 + src/hx/libs/std/Sys.cpp | 21 ++++++++++++++--- test/regression/SleepLongDuration/Main.hx | 24 ++++++++++++++++++++ test/regression/SleepLongDuration/build.hxml | 2 ++ test/regression/SleepLongDuration/stdout.txt | 5 ++++ test/regression/emscripten-skip.txt | 1 + 6 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 test/regression/SleepLongDuration/Main.hx create mode 100644 test/regression/SleepLongDuration/build.hxml create mode 100644 test/regression/SleepLongDuration/stdout.txt diff --git a/Changes.md b/Changes.md index 52acfb4e1..1c851e2ce 100644 --- a/Changes.md +++ b/Changes.md @@ -8,6 +8,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host * Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again +* Fixed Sys.sleep returning at once, or after a much shorter time, for a duration that did not fit the native call: on Windows from 2^32 milliseconds, about 49.7 days, and past 2^31 seconds elsewhere. Sys.sleep(Math.POSITIVE_INFINITY) returned at once instead of sleeping for good. A long sleep goes in parts that fit * Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there diff --git a/src/hx/libs/std/Sys.cpp b/src/hx/libs/std/Sys.cpp index 7e562d8a8..392642266 100644 --- a/src/hx/libs/std/Sys.cpp +++ b/src/hx/libs/std/Sys.cpp @@ -200,15 +200,29 @@ void _hx_std_sys_sleep( double f ) f = 0; hx::EnterGCFreeZone(); #if defined(NEKO_WINDOWS) - Sleep((DWORD)(f * 1000)); + // Sleep takes a DWORD of milliseconds, and 0xffffffff is INFINITE. A longer + // sleep, infinity included, converted to a short one or none, so it goes in + // parts that fit + double ms = f * 1000; + while (ms >= 4294967294.0) + { + Sleep(4294967294u); + ms -= 4294967294.0; + } + Sleep((DWORD)ms); #elif defined(EPPC) //TODO: Implement sys_sleep for EPPC #else + // In parts of up to 1e9 seconds, which fit an int: a longer sleep, infinity + // included, overflowed the conversion and returned at once + do { + double part = f < 1e9 ? f : 1e9; + f -= part; struct timespec t; struct timespec tmp; - t.tv_sec = (int)(f); - t.tv_nsec = (int)(((f) - t.tv_sec) * 1e9); + t.tv_sec = (int)(part); + t.tv_nsec = (int)((part - t.tv_sec) * 1e9); while( nanosleep(&t,&tmp) == -1 ) { if( errno != EINTR ) @@ -219,6 +233,7 @@ void _hx_std_sys_sleep( double f ) t = tmp; } } + while (f > 0); #endif hx::ExitGCFreeZone(); } diff --git a/test/regression/SleepLongDuration/Main.hx b/test/regression/SleepLongDuration/Main.hx new file mode 100644 index 000000000..d04b1e20f --- /dev/null +++ b/test/regression/SleepLongDuration/Main.hx @@ -0,0 +1,24 @@ +import sys.thread.Lock; +import sys.thread.Thread; + +// Sys.sleep returned at once for a duration that did not fit the native call: +// on Windows from 2^32 milliseconds, about 49.7 days, which wrapped to a short +// sleep, and on POSIX past 2^31 seconds. Infinity, a way to sleep for good, +// was one of them. Each sleep runs in a thread, which is still asleep later. +function main() { + for (seconds in [4294967.3, 3e9, Math.POSITIVE_INFINITY]) { + var woke = new Lock(); + Thread.create(() -> { + Sys.sleep(seconds); + woke.release(); + }); + Sys.println('Sys.sleep($seconds) still sleeping after 0.3 s: ${!woke.wait(0.3)}'); + } + var t0 = haxe.Timer.stamp(); + Sys.sleep(0.05); + Sys.println("Sys.sleep(0.05) took at least 0.05 s: " + (haxe.Timer.stamp() - t0 >= 0.049)); + Sys.sleep(0); + Sys.sleep(-1); + Sys.sleep(Math.NaN); + Sys.println("zero, negative and NaN return"); +} diff --git a/test/regression/SleepLongDuration/build.hxml b/test/regression/SleepLongDuration/build.hxml new file mode 100644 index 000000000..f35632dbc --- /dev/null +++ b/test/regression/SleepLongDuration/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-m Main diff --git a/test/regression/SleepLongDuration/stdout.txt b/test/regression/SleepLongDuration/stdout.txt new file mode 100644 index 000000000..2917ff925 --- /dev/null +++ b/test/regression/SleepLongDuration/stdout.txt @@ -0,0 +1,5 @@ +Sys.sleep(4294967.3) still sleeping after 0.3 s: true +Sys.sleep(3000000000) still sleeping after 0.3 s: true +Sys.sleep(Infinity) still sleeping after 0.3 s: true +Sys.sleep(0.05) took at least 0.05 s: true +zero, negative and NaN return diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index a0d1016bf..dd0bed045 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -94,6 +94,7 @@ RegisterHeldRoots: threads RootWalkRace: threads (std::thread, which needs a pthreads build) SafePointInGcFreeZone: threads SemaphoreLoopCollect: threads +SleepLongDuration: threads SocketEmptyBytes: sockets SocketIpv6Accept: sockets SocketIpv6ConcurrentFormat: threads (std::thread, which needs a pthreads build) From dcb752f941bbe760dcbca724d8a3ac2bf5f64ef9 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:56:15 -0500 Subject: [PATCH 10/31] Cap Socket.select's timeout at what a timeval holds init_timeval converted the timeout to an int of seconds, which overflowed past 2^31 s: Windows' select then returned at once with nothing ready, and Math.POSITIVE_INFINITY did the same. NaN went through the same undefined conversion. Socket.setTimeout already caps its own at 0x7fffffff seconds. The timeout is capped there too, and NaN waits for nothing. Windows' poll builds its timeval with the same function. Test: SocketSelectLongTimeout selects on a listener with timeouts of 3e9 s and infinity while another thread connects after 0.3 s. Before the change both returned at once with nothing ready on Windows x64; after it both wait for the connection, 5 of 5. --- Changes.md | 1 + src/hx/libs/std/Socket.cpp | 5 +++ .../SocketSelectLongTimeout/Main.hx | 32 +++++++++++++++++++ .../SocketSelectLongTimeout/build.hxml | 2 ++ .../SocketSelectLongTimeout/stdout.txt | 3 ++ test/regression/emscripten-skip.txt | 1 + 6 files changed, 44 insertions(+) create mode 100644 test/regression/SocketSelectLongTimeout/Main.hx create mode 100644 test/regression/SocketSelectLongTimeout/build.hxml create mode 100644 test/regression/SocketSelectLongTimeout/stdout.txt diff --git a/Changes.md b/Changes.md index 1c851e2ce..133fdb43a 100644 --- a/Changes.md +++ b/Changes.md @@ -8,6 +8,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host * Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again +* Fixed Socket.select returning at once with nothing ready for a timeout past 2^31 seconds, Math.POSITIVE_INFINITY included: the timeout overflowed an int of seconds. It is capped at 2^31-1 seconds, as Socket.setTimeout's is, and a NaN timeout waits for nothing. Windows' Poll, which takes the same timeout, is fixed with it * Fixed Sys.sleep returning at once, or after a much shorter time, for a duration that did not fit the native call: on Windows from 2^32 milliseconds, about 49.7 days, and past 2^31 seconds elsewhere. Sys.sleep(Math.POSITIVE_INFINITY) returned at once instead of sleeping for good. A long sleep goes in parts that fit * Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there diff --git a/src/hx/libs/std/Socket.cpp b/src/hx/libs/std/Socket.cpp index 2f652f92a..17be843d8 100644 --- a/src/hx/libs/std/Socket.cpp +++ b/src/hx/libs/std/Socket.cpp @@ -1302,6 +1302,11 @@ static void make_array_result_inplace(Array a, fd_set *tmp) static struct timeval *init_timeval( double f, struct timeval *t ) { if (f<0) return 0; + // The int conversions below overflowed past 2^31 seconds, and select + // returned at once with nothing ready: the wait is capped, as + // set_timeout caps its own. NaN waits for nothing + if (!(f < 0x7fffffff)) + f = f == f ? 0x7fffffff : 0; t->tv_usec = (int)( (f - (int)f ) * 1000000 ); t->tv_sec = (int)f; return t; diff --git a/test/regression/SocketSelectLongTimeout/Main.hx b/test/regression/SocketSelectLongTimeout/Main.hx new file mode 100644 index 000000000..ee5107a2d --- /dev/null +++ b/test/regression/SocketSelectLongTimeout/Main.hx @@ -0,0 +1,32 @@ +import sys.net.Host; +import sys.net.Socket; +import sys.thread.Thread; + +// Socket.select converted its timeout to an int of seconds, which overflowed +// past 2^31 s: a select with nothing ready returned at once instead of waiting. +// Infinity was one such timeout. Each select here waits for a connection that +// another thread makes after 0.3 s. +function main() { + var server = new Socket(); + server.bind(new Host("127.0.0.1"), 0); + server.listen(4); + var port = server.host().port; + for (timeout in [3e9, Math.POSITIVE_INFINITY]) { + Thread.create(() -> { + Sys.sleep(0.3); + var client = new Socket(); + client.connect(new Host("127.0.0.1"), port); + Sys.sleep(0.2); + client.close(); + }); + var t0 = haxe.Timer.stamp(); + var ready = Socket.select([server], [], [], timeout).read.length; + var waited = haxe.Timer.stamp() - t0 >= 0.25; + Sys.println('select with a timeout of $timeout waited for the connection: ${waited && ready == 1}'); + server.accept().close(); + } + var t0 = haxe.Timer.stamp(); + var ready = Socket.select([server], [], [], Math.NaN).read.length; + Sys.println("select with a NaN timeout returned at once: " + (ready == 0 && haxe.Timer.stamp() - t0 < 0.25)); + server.close(); +} diff --git a/test/regression/SocketSelectLongTimeout/build.hxml b/test/regression/SocketSelectLongTimeout/build.hxml new file mode 100644 index 000000000..f35632dbc --- /dev/null +++ b/test/regression/SocketSelectLongTimeout/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-m Main diff --git a/test/regression/SocketSelectLongTimeout/stdout.txt b/test/regression/SocketSelectLongTimeout/stdout.txt new file mode 100644 index 000000000..dcb173bea --- /dev/null +++ b/test/regression/SocketSelectLongTimeout/stdout.txt @@ -0,0 +1,3 @@ +select with a timeout of 3000000000 waited for the connection: true +select with a timeout of Infinity waited for the connection: true +select with a NaN timeout returned at once: true diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index dd0bed045..4ac328782 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -100,6 +100,7 @@ SocketIpv6Accept: sockets SocketIpv6ConcurrentFormat: threads (std::thread, which needs a pthreads build) SocketLibraryHandle: sockets SocketReadAhead: sockets +SocketSelectLongTimeout: sockets SocketTinyTimeout: sockets SqliteNativeMemoryPressure: threads (std::thread, which needs a pthreads build) SqliteResultConnectionLifetime: threads (std::thread, which needs a pthreads build) From fac3dc2964eadc41bd8b6414acfc1a2590c2b9fc Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 07:24:18 -0500 Subject: [PATCH 11/31] Make a thread's scratch buffer only at API level 500 makeArrays gave every thread object a 65,535-byte scratch array, which only hx::thread::Scratch reads, and Scratch.cpp is compiled only at API level 500. With Haxe 4 every Haxe thread kept one for as long as it ran. A native thread that calls sys.thread.Thread.current() is added to Haxe 4's thread list and never removed, so its 64 KB stayed until the process ended: 2,000 such threads held 128 MB. Below API level 500 the array is no longer made. Marking and visiting already accept a null array. Test: ThreadScratchApi4 measures what 64 running threads hold after a collection. Each held more than 16 KB before the change and less after it, 5 of 5, Windows x64, MSVC 19.44. --- Changes.md | 1 + src/hx/thread/ThreadImpl.cpp | 3 ++ test/regression/ThreadScratchApi4/Main.hx | 34 ++++++++++++++++++++ test/regression/ThreadScratchApi4/build.hxml | 2 ++ test/regression/ThreadScratchApi4/stdout.txt | 1 + test/regression/emscripten-skip.txt | 1 + 6 files changed, 42 insertions(+) create mode 100644 test/regression/ThreadScratchApi4/Main.hx create mode 100644 test/regression/ThreadScratchApi4/build.hxml create mode 100644 test/regression/ThreadScratchApi4/stdout.txt diff --git a/Changes.md b/Changes.md index 133fdb43a..5d7058062 100644 --- a/Changes.md +++ b/Changes.md @@ -8,6 +8,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed `haxelib run hxcpp defines` failing with "Could not read" from any folder but tools/hxcpp: it opened the build tool's sources by name, relative to the caller's folder. It lists the defines the build tool reads again * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host * Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again +* Each thread object no longer holds a 64 KB buffer below API level 500: only Haxe 5's hx::thread::Scratch uses it, yet with Haxe 4 every Haxe thread kept one while it ran, and so did every native thread that asked for its sys.thread.Thread, for as long as Haxe's own thread list holds it, which for a native thread is until the process ends * Fixed Socket.select returning at once with nothing ready for a timeout past 2^31 seconds, Math.POSITIVE_INFINITY included: the timeout overflowed an int of seconds. It is capped at 2^31-1 seconds, as Socket.setTimeout's is, and a NaN timeout waits for nothing. Windows' Poll, which takes the same timeout, is fixed with it * Fixed Sys.sleep returning at once, or after a much shorter time, for a duration that did not fit the native call: on Windows from 2^32 milliseconds, about 49.7 days, and past 2^31 seconds elsewhere. Sys.sleep(Math.POSITIVE_INFINITY) returned at once instead of sleeping for good. A long sleep goes in parts that fit * Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did diff --git a/src/hx/thread/ThreadImpl.cpp b/src/hx/thread/ThreadImpl.cpp index 0c5e76106..c9d8db617 100644 --- a/src/hx/thread/ThreadImpl.cpp +++ b/src/hx/thread/ThreadImpl.cpp @@ -168,7 +168,10 @@ int hx::thread::Thread_obj::id() void hx::thread::ThreadImpl_obj::makeArrays() { HX_INCREMENTAL_MUTATION(this); +#if (HXCPP_API_LEVEL >= 500) + // Only hx::thread::Scratch, which Haxe 5 uses, reads it scratch = Array(std::numeric_limits::max(), std::numeric_limits::max()); +#endif slots = Array(0, 0); localSlots = Array(0, 0); localGenerations = Array(0, 0); diff --git a/test/regression/ThreadScratchApi4/Main.hx b/test/regression/ThreadScratchApi4/Main.hx new file mode 100644 index 000000000..6ce8b77d9 --- /dev/null +++ b/test/regression/ThreadScratchApi4/Main.hx @@ -0,0 +1,34 @@ +import cpp.vm.Gc; +import sys.thread.Lock; +import sys.thread.Thread; + +// Each thread object held a 64 KB scratch buffer, which only Haxe 5's +// hx::thread::Scratch uses. Below API level 500 nothing reads it, yet every +// Haxe thread, and every native thread that asked for its Thread, kept one. +function main() { + #if (hxcpp_api_level >= 500) + Sys.println("each running thread holds less than 16 KB: true"); + #else + var count = 64; + Gc.run(true); + var before = Gc.memInfo64(Gc.MEM_INFO_USAGE); + var started = new Lock(); + var stop = new Lock(); + var stopped = new Lock(); + for (i in 0...count) + Thread.create(() -> { + started.release(); + stop.wait(); + stopped.release(); + }); + for (i in 0...count) + started.wait(); + Gc.run(true); + var perThread = (Gc.memInfo64(Gc.MEM_INFO_USAGE) - before) / count; + Sys.println("each running thread holds less than 16 KB: " + (perThread < 16384)); + for (i in 0...count) + stop.release(); + for (i in 0...count) + stopped.wait(); + #end +} diff --git a/test/regression/ThreadScratchApi4/build.hxml b/test/regression/ThreadScratchApi4/build.hxml new file mode 100644 index 000000000..f35632dbc --- /dev/null +++ b/test/regression/ThreadScratchApi4/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-m Main diff --git a/test/regression/ThreadScratchApi4/stdout.txt b/test/regression/ThreadScratchApi4/stdout.txt new file mode 100644 index 000000000..8ee82030c --- /dev/null +++ b/test/regression/ThreadScratchApi4/stdout.txt @@ -0,0 +1 @@ +each running thread holds less than 16 KB: true diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index 4ac328782..6342a8d49 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -114,6 +114,7 @@ SslSniName: sockets ThreadEndsAtExit: threads ThreadEndsAtExitDebugger: threads ThreadEndsWhileCollecting: threads +ThreadScratchApi4: threads ThreadWaitTimeouts: threads TimerStampFirstCall: threads UncaughtNullException: processes (runs itself) From 190b75cfa2bb931f3a8b91c9054cf2294158d3e0 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:48:05 -0500 Subject: [PATCH 12/31] Refuse a cppia super constructor call into a class with no constructor The fuzz job's cppia target stopped in CallMember::linkSuperCall with "member access within null pointer of type 'CppiaFunction'": a super constructor call named a script class with no constructor, in it or in a script class above it, and linking read the constructor's function through its null newFunc. Haxe only calls a constructor a class has or inherits, and linkTypes gives a script class without one its script super class's, so only a malformed file gets there. Such a call now fails the load with "Super constructor call on a class with no constructor". Tests: CppiaMalformedConstructors loads such files, interpreted and with the JIT, and runs scripts whose classes take a constructor from a script class and from a host class above them; the fuzz input is in test/fuzz/regressions/cppia. openfl/libhxcpp#257. --- Changes.md | 1 + src/hx/cppia/Cppia.cpp | 5 ++ ...uper-call-into-a-class-with-no-constructor | Bin 0 -> 781 bytes .../CppiaMalformedConstructors/HostBase.hx | 9 ++++ .../CppiaMalformedConstructors/Main.hx | 49 ++++++++++++++++++ .../CppiaMalformedConstructors/Script.hx | 36 +++++++++++++ .../CppiaMalformedConstructors/build.hxml | 7 +++ .../CppiaMalformedConstructors/stdout.txt | 21 ++++++++ 8 files changed, 128 insertions(+) create mode 100644 test/fuzz/regressions/cppia/super-call-into-a-class-with-no-constructor create mode 100644 test/regression/CppiaMalformedConstructors/HostBase.hx create mode 100644 test/regression/CppiaMalformedConstructors/Main.hx create mode 100644 test/regression/CppiaMalformedConstructors/Script.hx create mode 100644 test/regression/CppiaMalformedConstructors/build.hxml create mode 100644 test/regression/CppiaMalformedConstructors/stdout.txt diff --git a/Changes.md b/Changes.md index 5d7058062..f9808d097 100644 --- a/Changes.md +++ b/Changes.md @@ -13,6 +13,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed Sys.sleep returning at once, or after a much shorter time, for a duration that did not fit the native call: on Windows from 2^32 milliseconds, about 49.7 days, and past 2^31 seconds elsewhere. Sys.sleep(Math.POSITIVE_INFINITY) returned at once instead of sleeping for good. A long sleep goes in parts that fit * Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there +* Fixed a malformed cppia file that calls the super constructor of a class with no constructor, in it or in a script class above it, crashing the loader, which read the constructor through a null pointer. It fails to load with "Super constructor call on a class with no constructor". The fuzz job found it (libhxcpp #257) libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index af0be4a78..02f6855de 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -3530,6 +3530,11 @@ struct CallMember : public CppiaExpr if (type->cppiaClass) { + // Haxe only calls a constructor a class has or inherits, and a script + // class without one has its script super class's, so only a + // malformed file calls one where neither has any + if (!type->cppiaClass->newFunc) + throw "Super constructor call on a class with no constructor"; //CPPIALOG("Using cppia super %p %p\n", type->cppiaClass->newFunc, type->cppiaClass->newFunc->funExpr); CppiaExpr *replace = new CallFunExpr( this, 0, (ScriptCallable*)type->cppiaClass->newFunc->funExpr, args, true ); replace->link(inModule); diff --git a/test/fuzz/regressions/cppia/super-call-into-a-class-with-no-constructor b/test/fuzz/regressions/cppia/super-call-into-a-class-with-no-constructor new file mode 100644 index 0000000000000000000000000000000000000000..4df89ef4a74cb3326e80c93fc73f76aa15ca1906 GIT binary patch literal 781 zcmZ8dO;6iE5Z&3tG1+`jLRqK;(H2k-RWT%d)rwS=77nOTkoW<}iC2kHVk3KLL)CMy zMHE%N^tbU(tiBDX2PA7eGw;ot+4u4E^zegd?}!a0HdU;@iY*lfBQ3U7Kh@fZUDZpj zVxv>BqXv37Oq6IT1=c8tV$oLTedL!{*#PYf=`hL+lt#fN;(5P?v7s0D%>c=YHixl+ zv`K@WT``dcaMnwsOCws_D$_d5)*otbFb0+eahQyt{7t8c*k(bR2G{nitMt2u+O8S| z*=IdApv3mq+Om)NFN(yZQ5*vPl0+e{iZIn`t34~^opdjB54DvUYcI^aGm}Me{}>N= z`_MZ|`j9&cM%^&L_?Zbs`w@y@Okk2U^CoJCkEQUf`tKj!UVM$c-6R>}&i!jR8}&q6 z>=R=S(R(7|jJpNm`A^9>D^q=m3OxTKIXgIDoHGy$oVkGb>Y_vITx;ENmWX#si&W$U zRuvdQx+YSMpmsZl_RH^Iwvc zN^2$DfU=hVmDuUCO-eZOrc>wOmkYo|ekXCtGDdHu1y^in6sI|dNCS`IU^IFCRg=)V zlhb_BeKtEaGr`|&%*uwla52lDlh1v)@v<|wFD6zo5+qp#R>}N+j#Y3=eM!vw3=x~| w=O%{93QUA0^F?diLY28ysH|5LP@5G?Mf*jKxi#CaEmzQ41@>u%Tz1U<0byyYr~m)} literal 0 HcmV?d00001 diff --git a/test/regression/CppiaMalformedConstructors/HostBase.hx b/test/regression/CppiaMalformedConstructors/HostBase.hx new file mode 100644 index 000000000..6cdebfb2c --- /dev/null +++ b/test/regression/CppiaMalformedConstructors/HostBase.hx @@ -0,0 +1,9 @@ +// A host class for the script's classes to extend. The script is compiled with +// its own copy, which the host's replaces when it loads +class HostBase { + public var made:String; + + public function new(n:Int) { + made = #if cppia 'script\'s HostBase $n' #else 'HostBase $n' #end; + } +} diff --git a/test/regression/CppiaMalformedConstructors/Main.hx b/test/regression/CppiaMalformedConstructors/Main.hx new file mode 100644 index 000000000..2c853538e --- /dev/null +++ b/test/regression/CppiaMalformedConstructors/Main.hx @@ -0,0 +1,49 @@ +// A super constructor call is linked to the constructor of the class it +// names. A class with none, in it or in a script class above it, has none to +// link to, which only a malformed cppia file asks for, and the loader read +// through the null constructor. Such a file fails to load now. Scripts whose +// classes take a constructor from above, from a script class or from the +// host, still load and run. +class Main { + // Strings "new" and "main"; types Foo and Bar; Foo.main's body is the + // expression given + static function withMain(main:String, bar:String) { + return 'CPPIA 2 3 new 4 main 3 0 3 Foo 3 Bar 2 CLASS 1 0 0 1 FUNCTION 1 0 1 0 0 0 0 FUN 0 0 0 0 $main $bar NOMAIN RESOURCES 0 '; + } + + // Bar alone, and a main that makes one + static function newBar(bar:String) { + return 'CPPIA 2 3 new 4 main 3 0 3 Foo 3 Bar 1 $bar MAIN 0 0 NEW 2 0 RESOURCES 0 '; + } + + static function check(name:String, code:String) { + try { + var module = cpp.cppia.Module.fromString(code); + module.boot(); + module.run(); + Sys.println('$name loaded and ran'); + untyped module.release(); + } catch (e:Dynamic) { + Sys.println('$name threw: $e'); + } + } + + static function main() { + Sys.println(new HostBase(0).made); + var script = haxe.io.Path.join([haxe.io.Path.directory(Sys.programPath()), "script.cppia"]); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + Sys.println(jit ? "jit" : "interpreted"); + check("super call into a class with no constructor", withMain("CALLSUPERNEW 2 0", "CLASS 2 0 0 0")); + check("super call into an interface", withMain("CALLSUPERNEW 2 0", "INTERFACE 2 0 0 0")); + check("super call into a constructor", withMain("CALLSUPERNEW 2 0", "CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 FUN 0 0 0 0 BLOCK 0")); + check("new of a class with no constructor", newBar("CLASS 2 0 0 0")); + check("new of a constructor", newBar("CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 FUN 0 0 0 0 BLOCK 0")); + try { + cpp.cppia.Host.runFile(script); + } catch (e:Dynamic) { + Sys.println("script threw: " + e); + } + } + } +} diff --git a/test/regression/CppiaMalformedConstructors/Script.hx b/test/regression/CppiaMalformedConstructors/Script.hx new file mode 100644 index 000000000..70ce7334f --- /dev/null +++ b/test/regression/CppiaMalformedConstructors/Script.hx @@ -0,0 +1,36 @@ +// Classes that take their constructor from above: Middle from Top, a script +// class, and HostMiddle from Main's HostBase, a host class +class Top { + public var made:String; + + public function new(n:Int) { + made = 'Top $n'; + } +} + +class Middle extends Top {} + +class Bottom extends Middle { + public function new() { + super(1); + made += " then Bottom"; + } +} + +class HostMiddle extends HostBase {} + +class HostBottom extends HostMiddle { + public function new() { + super(2); + made += " then HostBottom"; + } +} + +class Script { + static function main() { + Sys.println(new Bottom().made); + Sys.println(new Middle(3).made); + Sys.println(new HostBottom().made); + Sys.println(new HostMiddle(4).made); + } +} diff --git a/test/regression/CppiaMalformedConstructors/build.hxml b/test/regression/CppiaMalformedConstructors/build.hxml new file mode 100644 index 000000000..d20d36bd4 --- /dev/null +++ b/test/regression/CppiaMalformedConstructors/build.hxml @@ -0,0 +1,7 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaMalformedConstructors/stdout.txt b/test/regression/CppiaMalformedConstructors/stdout.txt new file mode 100644 index 000000000..c93e7fa7a --- /dev/null +++ b/test/regression/CppiaMalformedConstructors/stdout.txt @@ -0,0 +1,21 @@ +HostBase 0 +interpreted +super call into a class with no constructor threw: Super constructor call on a class with no constructor +super call into an interface threw: Super constructor call on a class with no constructor +super call into a constructor loaded and ran +new of a class with no constructor loaded and ran +new of a constructor loaded and ran +Top 1 then Bottom +Top 3 +HostBase 2 then HostBottom +HostBase 4 +jit +super call into a class with no constructor threw: Super constructor call on a class with no constructor +super call into an interface threw: Super constructor call on a class with no constructor +super call into a constructor loaded and ran +new of a class with no constructor loaded and ran +new of a constructor loaded and ran +Top 1 then Bottom +Top 3 +HostBase 2 then HostBottom +HostBase 4 From a896acfdbfa3544de5ff6d720f790da8de8763cb Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:49:09 -0500 Subject: [PATCH 13/31] Refuse a cppia constructor that is not a function A function's funExpr is its body only when the body is a function, so a constructor whose body is a bare RETURN, which a malformed file can give, or an interface's "new", which has no body, has none. linkTypes took such a function as the class's constructor anyway. A super call into it linked a CallFunExpr that read the return type through the null function, crashing the loader as the super call into a class with no constructor did, and a new of the class loaded and then crashed when it ran, interpreted and under the JIT. Haxe gives a class only a constructor with a body and an interface none, so such a file now fails to load with "Constructor is not a function". Tests: CppiaMalformedConstructors loads the files, interpreted and with the JIT; the fuzz input with the super call is in test/fuzz/regressions/cppia. openfl/libhxcpp#257. --- Changes.md | 1 + src/hx/cppia/CppiaClasses.cpp | 4 ++++ .../super-call-into-a-constructor-that-is-a-return | 1 + test/regression/CppiaMalformedConstructors/Main.hx | 11 ++++++++--- test/regression/CppiaMalformedConstructors/stdout.txt | 6 ++++++ 5 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 test/fuzz/regressions/cppia/super-call-into-a-constructor-that-is-a-return diff --git a/Changes.md b/Changes.md index f9808d097..f875783a4 100644 --- a/Changes.md +++ b/Changes.md @@ -14,6 +14,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there * Fixed a malformed cppia file that calls the super constructor of a class with no constructor, in it or in a script class above it, crashing the loader, which read the constructor through a null pointer. It fails to load with "Super constructor call on a class with no constructor". The fuzz job found it (libhxcpp #257) +* Fixed a malformed cppia file whose constructor was a bare return or, in an interface, had no body loading and then crashing: a super call into it crashed the loader, and a new of its class crashed when it ran, through the constructor's missing function. It fails to load with "Constructor is not a function" (libhxcpp #257) libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/CppiaClasses.cpp b/src/hx/cppia/CppiaClasses.cpp index 03e6298c3..3b30bbfdc 100644 --- a/src/hx/cppia/CppiaClasses.cpp +++ b/src/hx/cppia/CppiaClasses.cpp @@ -1415,6 +1415,10 @@ void CppiaClassInfo::linkTypes() { if (staticFunctions[i]->name == "new") { + // A super call or a new runs it as a function, but a malformed file + // can give a bare RETURN for its body, or an interface's no body + if (!staticFunctions[i]->funExpr) + throw "Constructor is not a function"; newFunc = staticFunctions[i]; staticFunctions.erase( staticFunctions.begin() + i); } diff --git a/test/fuzz/regressions/cppia/super-call-into-a-constructor-that-is-a-return b/test/fuzz/regressions/cppia/super-call-into-a-constructor-that-is-a-return new file mode 100644 index 000000000..4b2e97f90 --- /dev/null +++ b/test/fuzz/regressions/cppia/super-call-into-a-constructor-that-is-a-return @@ -0,0 +1 @@ +CPPIA 2 3 new 4 main 3 0 3 Foo 3 Bar 2 CLASS 1 0 0 1 FUNCTION 1 0 1 0 0 0 0 FUN 0 0 0 0 CALLSUPERNEW 2 0 CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 RETURN NOMAIN RESOURCES 0 \ No newline at end of file diff --git a/test/regression/CppiaMalformedConstructors/Main.hx b/test/regression/CppiaMalformedConstructors/Main.hx index 2c853538e..9bbf11ce3 100644 --- a/test/regression/CppiaMalformedConstructors/Main.hx +++ b/test/regression/CppiaMalformedConstructors/Main.hx @@ -1,9 +1,11 @@ // A super constructor call is linked to the constructor of the class it // names. A class with none, in it or in a script class above it, has none to // link to, which only a malformed cppia file asks for, and the loader read -// through the null constructor. Such a file fails to load now. Scripts whose -// classes take a constructor from above, from a script class or from the -// host, still load and run. +// through the null constructor. A constructor whose body is not a function, +// as a malformed file can give, loaded, and a super call into it crashed the +// loader the same way, and a new of its class crashed when it ran. Such files +// fail to load now. Scripts whose classes take a constructor from above, from +// a script class or from the host, still load and run. class Main { // Strings "new" and "main"; types Foo and Bar; Foo.main's body is the // expression given @@ -36,7 +38,10 @@ class Main { Sys.println(jit ? "jit" : "interpreted"); check("super call into a class with no constructor", withMain("CALLSUPERNEW 2 0", "CLASS 2 0 0 0")); check("super call into an interface", withMain("CALLSUPERNEW 2 0", "INTERFACE 2 0 0 0")); + check("super call into a constructor that is a return", withMain("CALLSUPERNEW 2 0", "CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 RETURN")); + check("super call into an interface with a constructor", withMain("CALLSUPERNEW 2 0", "INTERFACE 2 0 0 1 FUNCTION 1 0 0 0 0")); check("super call into a constructor", withMain("CALLSUPERNEW 2 0", "CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 FUN 0 0 0 0 BLOCK 0")); + check("new of a constructor that is a return", newBar("CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 RETURN")); check("new of a class with no constructor", newBar("CLASS 2 0 0 0")); check("new of a constructor", newBar("CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 FUN 0 0 0 0 BLOCK 0")); try { diff --git a/test/regression/CppiaMalformedConstructors/stdout.txt b/test/regression/CppiaMalformedConstructors/stdout.txt index c93e7fa7a..cdc826373 100644 --- a/test/regression/CppiaMalformedConstructors/stdout.txt +++ b/test/regression/CppiaMalformedConstructors/stdout.txt @@ -2,7 +2,10 @@ HostBase 0 interpreted super call into a class with no constructor threw: Super constructor call on a class with no constructor super call into an interface threw: Super constructor call on a class with no constructor +super call into a constructor that is a return threw: Constructor is not a function +super call into an interface with a constructor threw: Constructor is not a function super call into a constructor loaded and ran +new of a constructor that is a return threw: Constructor is not a function new of a class with no constructor loaded and ran new of a constructor loaded and ran Top 1 then Bottom @@ -12,7 +15,10 @@ HostBase 4 jit super call into a class with no constructor threw: Super constructor call on a class with no constructor super call into an interface threw: Super constructor call on a class with no constructor +super call into a constructor that is a return threw: Constructor is not a function +super call into an interface with a constructor threw: Constructor is not a function super call into a constructor loaded and ran +new of a constructor that is a return threw: Constructor is not a function new of a class with no constructor loaded and ran new of a constructor loaded and ran Top 1 then Bottom From 82f8794acce178d7d15f13454672eee811d353e2 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:49:49 -0500 Subject: [PATCH 14/31] Refuse a new instance of a cppia script interface A new of a type with a script class linked without a check, but a script interface has no host base: running the new made the object through the interface's null haxeBase, interpreted and under the JIT. Under the generational collector the JIT also reads the base's data offset as it compiles the new. Haxe makes no instance of an interface, so only a malformed file asks for one. Such a new is linked as one of a type with nothing to construct, and the file fails to load with "Could not link new" and the interface's name. Tests: CppiaMalformedConstructors loads such a file, interpreted and with the JIT. openfl/libhxcpp#257. --- Changes.md | 1 + src/hx/cppia/Cppia.cpp | 7 ++++--- test/regression/CppiaMalformedConstructors/Main.hx | 8 +++++--- test/regression/CppiaMalformedConstructors/stdout.txt | 2 ++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Changes.md b/Changes.md index f875783a4..f641bc037 100644 --- a/Changes.md +++ b/Changes.md @@ -15,6 +15,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there * Fixed a malformed cppia file that calls the super constructor of a class with no constructor, in it or in a script class above it, crashing the loader, which read the constructor through a null pointer. It fails to load with "Super constructor call on a class with no constructor". The fuzz job found it (libhxcpp #257) * Fixed a malformed cppia file whose constructor was a bare return or, in an interface, had no body loading and then crashing: a super call into it crashed the loader, and a new of its class crashed when it ran, through the constructor's missing function. It fails to load with "Constructor is not a function" (libhxcpp #257) +* Fixed a malformed cppia file that makes a new instance of a script interface loading and then crashing when it ran, interpreted and under the JIT: an interface has no host base to make the object from. It fails to load with "Could not link new" and the interface's name, as for a type with nothing to construct (libhxcpp #257) libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index 02f6855de..b08789f11 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -1692,9 +1692,10 @@ struct NewExpr : public CppiaDynamicExpr if (!type->cppiaClass && type->haxeClass.mPtr) constructor = type->haxeClass.mPtr->mConstructArgs; - // Nothing to construct: runObject and genCode would dereference the null cppiaClass. - // The module never runs: linking goes on to find anything else missing - if (!type->arrayType && !constructor && !type->cppiaClass) + // Nothing to construct: runObject and genCode would dereference the null cppiaClass, + // or a script interface's null haxeBase. The module never runs: linking goes on to + // find anything else missing + if (!type->arrayType && !constructor && (!type->cppiaClass || !type->cppiaClass->haxeBase)) inModule.linkMiss(this, std::string("new ") + type->name.c_str()); LinkExpressions(args,inModule); diff --git a/test/regression/CppiaMalformedConstructors/Main.hx b/test/regression/CppiaMalformedConstructors/Main.hx index 9bbf11ce3..75e2fcece 100644 --- a/test/regression/CppiaMalformedConstructors/Main.hx +++ b/test/regression/CppiaMalformedConstructors/Main.hx @@ -3,9 +3,10 @@ // link to, which only a malformed cppia file asks for, and the loader read // through the null constructor. A constructor whose body is not a function, // as a malformed file can give, loaded, and a super call into it crashed the -// loader the same way, and a new of its class crashed when it ran. Such files -// fail to load now. Scripts whose classes take a constructor from above, from -// a script class or from the host, still load and run. +// loader the same way, and a new of its class crashed when it ran, as a new of +// a script interface did. Such files fail to load now. Scripts whose classes +// take a constructor from above, from a script class or from the host, still +// load and run. class Main { // Strings "new" and "main"; types Foo and Bar; Foo.main's body is the // expression given @@ -42,6 +43,7 @@ class Main { check("super call into an interface with a constructor", withMain("CALLSUPERNEW 2 0", "INTERFACE 2 0 0 1 FUNCTION 1 0 0 0 0")); check("super call into a constructor", withMain("CALLSUPERNEW 2 0", "CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 FUN 0 0 0 0 BLOCK 0")); check("new of a constructor that is a return", newBar("CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 RETURN")); + check("new of an interface", newBar("INTERFACE 2 0 0 0")); check("new of a class with no constructor", newBar("CLASS 2 0 0 0")); check("new of a constructor", newBar("CLASS 2 0 0 1 FUNCTION 1 0 0 0 0 0 0 FUN 0 0 0 0 BLOCK 0")); try { diff --git a/test/regression/CppiaMalformedConstructors/stdout.txt b/test/regression/CppiaMalformedConstructors/stdout.txt index cdc826373..4b4b161e4 100644 --- a/test/regression/CppiaMalformedConstructors/stdout.txt +++ b/test/regression/CppiaMalformedConstructors/stdout.txt @@ -6,6 +6,7 @@ super call into a constructor that is a return threw: Constructor is not a funct super call into an interface with a constructor threw: Constructor is not a function super call into a constructor loaded and ran new of a constructor that is a return threw: Constructor is not a function +new of an interface threw: Could not link new Bar new of a class with no constructor loaded and ran new of a constructor loaded and ran Top 1 then Bottom @@ -19,6 +20,7 @@ super call into a constructor that is a return threw: Constructor is not a funct super call into an interface with a constructor threw: Constructor is not a function super call into a constructor loaded and ran new of a constructor that is a return threw: Constructor is not a function +new of an interface threw: Could not link new Bar new of a class with no constructor loaded and ran new of a constructor loaded and ran Top 1 then Bottom From ec165c2f5307bd3da85611a2f3a17d140a27b75f Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:51:32 -0500 Subject: [PATCH 15/31] Lower the fuzz ereg target's match limit to 100,000 steps The fuzz job's ereg target reported a 12 second unit on run 44100276248, past libFuzzer's 10 second slow-unit report and 8 seconds short of the 20 second timeout. Its first match succeeds at once, so the target's stop after a match that gives up did not apply: split, replace and map each match again after that match, and each reached PCRE2's match limit of 10 million steps from one start before throwing. A pattern of adjacent \S* runs, on which a scheduled run timed out before, takes most of that limit at a start and still matches, so it is slow without ever reaching it: it replays in 5 seconds on MSVC without sanitizers. The limit is per start, so the pattern now begins with (*LIMIT_MATCH=100000), which a pattern can only lower. A pattern that does not backtrack takes a few steps a character: "^(a|b)*c$" over a 4,095 character subject needs between 8,000 and 20,000, and the other linear patterns tried over such a subject needed less. A pattern that backtracks past the limit throws, which the target still covers. The runtime keeps PCRE2's default limit. The approach and the \S* input come from an earlier fix that never reached main. Tests: both inputs are in test/fuzz/regressions/ereg, which the fuzz run and run.sh --replay go through. Without sanitizers, their replays take 1.5 and 5.1 seconds before and 0.2 to 0.4 after. openfl/libhxcpp#257. --- test/fuzz/Fuzz.hx | 18 ++++++++++++------ .../ereg/adjacent-repeats-under-match-limit | Bin 0 -> 90 bytes .../ereg/split-replace-and-map-at-match-limit | Bin 0 -> 145 bytes 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 test/fuzz/regressions/ereg/adjacent-repeats-under-match-limit create mode 100644 test/fuzz/regressions/ereg/split-replace-and-map-at-match-limit diff --git a/test/fuzz/Fuzz.hx b/test/fuzz/Fuzz.hx index e9b62970c..3a5308333 100644 --- a/test/fuzz/Fuzz.hx +++ b/test/fuzz/Fuzz.hx @@ -424,17 +424,23 @@ class Fuzz { var zero = 0; while (zero < rest.length && rest.get(zero) != 0) zero++; - var pattern = rest.getString(0, zero); + // PCRE2 gives up on a match after its match limit of steps from one + // start, and the EReg throws. Its default, 10 million, takes seconds + // under the sanitizers, and a pattern that backtracks through every way + // to split the subject, such as "(\D+)*x", reaches it. Split, replace + // and map match again after each match, so one input can reach it + // several times, or take nearly that long at several starts. The + // pattern lowers it to 100,000 steps: a pattern that does not backtrack + // takes a few steps a character, and "^(a|b)*c$" over a subject of + // max_len, 4096 bytes, needs fewer than 20,000. A limit in the input's + // own pattern can only lower it further + var pattern = "(*LIMIT_MATCH=100000)" + rest.getString(0, zero); var subject = zero < rest.length ? rest.getString(zero + 1, rest.length - zero - 1) : ""; var r:EReg = attempt(() -> new EReg(pattern, opts)); if (r == null) return; var matched:Null = attempt(() -> r.match(subject)); - // PCRE2 gives up on a match after its match limit, 10 million steps - // from one start, and the EReg throws. A pattern such as "(\D+)*x" - // backtracks through every way to split the subject, so each call over - // the subject below would run to the limit again: five of them took - // 12 seconds under the sanitizers + // A match that reached the limit would mostly reach it again below if (matched == null) return; if (matched) { diff --git a/test/fuzz/regressions/ereg/adjacent-repeats-under-match-limit b/test/fuzz/regressions/ereg/adjacent-repeats-under-match-limit new file mode 100644 index 0000000000000000000000000000000000000000..affcf252c37fcd6a7e27472ce2b08a23872b0132 GIT binary patch literal 90 fcmYdd)QSm)A{cGImVrE=sc`PE;4S;>_dyf@7$ptf literal 0 HcmV?d00001 diff --git a/test/fuzz/regressions/ereg/split-replace-and-map-at-match-limit b/test/fuzz/regressions/ereg/split-replace-and-map-at-match-limit new file mode 100644 index 0000000000000000000000000000000000000000..0bb5bcc7d701ba12cc3c803d4c2b2ea35f9131b4 GIT binary patch literal 145 zcmY$^h;h-@)Y3?-`TzgFqUN!=OT*?wC??j`MC;bX+Q%R`7~()_`&fJm;HK2vPAz8; aFl10nWMp7q1L}hbFo5iUa5xzlAWQ(JFf)q) literal 0 HcmV?d00001 From a8751c026abaca930d10afff9583fa40bbc23346 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:52:26 -0500 Subject: [PATCH 16/31] Compare Dates past 2^53 ms loosely in the fuzz serializer target A scheduled fuzz run's serializer target reported a Date of 2.16e18 ms changing in its last digit between the second and third round: the std's cpp Date keeps seconds, and past 2^53 ms, where a millisecond is less than a step between doubles, whole milliseconds can drift in a conversion, as fractions of one already could. That is the std's Date, not the runtime. canon() leaves such a Date's milliseconds out; a NaN time still compares as before. The 18 Dates from 2^50 to 1.2e16 ms that were tried, either side of 2^53, all settled after one round, so the cut-off stays at 2^53. From an earlier fix that never reached main, with the comparison turned round so a NaN time is not taken for a far one. Tests: the input is in test/fuzz/regressions/serializer, which failed on main with "serialized v2.1606691160662124e+18 then v2.1606691160662126e+18" and replays now. openfl/libhxcpp#257. --- test/fuzz/Fuzz.hx | 8 ++++++-- test/fuzz/regressions/serializer/date-past-2-to-53-ms | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 test/fuzz/regressions/serializer/date-past-2-to-53-ms diff --git a/test/fuzz/Fuzz.hx b/test/fuzz/Fuzz.hx index 3a5308333..ba67c1eff 100644 --- a/test/fuzz/Fuzz.hx +++ b/test/fuzz/Fuzz.hx @@ -611,8 +611,12 @@ class Fuzz { Type.getEnumName(e) + "." + Type.enumConstructor(v) + "(" + [for (p in Type.enumParameters(v)) canon(p, depth + 1)].join(",") + ")"; case TClass(String): haxe.Json.stringify(v); case TClass(Array): "[" + [for (x in (v : Array)) canon(x, depth + 1)].join(",") + "]"; - // The std's cpp Date keeps seconds, so a fraction of a millisecond drifts - case TClass(Date): "Date(" + Math.round((v : Date).getTime()) + ")"; + // The std's cpp Date keeps seconds, so a fraction of a millisecond + // drifts, and past 2^53 ms, where a millisecond is less than a step + // between doubles, so do whole milliseconds + case TClass(Date): + var t = (v : Date).getTime(); + Math.abs(t) >= 9007199254740992.0 ? "Date(beyond 2^53 ms)" : "Date(" + Math.round(t) + ")"; case TClass(haxe.io.Bytes): "Bytes(" + (v : Bytes).toHex() + ")"; case TClass(List): "List(" + [for (x in (v : List)) canon(x, depth + 1)].join(",") + ")"; case TClass(c) if (Std.isOfType(v, haxe.Constraints.IMap)): diff --git a/test/fuzz/regressions/serializer/date-past-2-to-53-ms b/test/fuzz/regressions/serializer/date-past-2-to-53-ms new file mode 100644 index 000000000..99c7080ed --- /dev/null +++ b/test/fuzz/regressions/serializer/date-past-2-to-53-ms @@ -0,0 +1 @@ +v2160669116066212160:aoy0:6[ \ No newline at end of file From 532668f287a1388ad622c5acc9bc65debeaa2fbc Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:57:43 -0500 Subject: [PATCH 17/31] Refuse a cppia dynamic function that is not a function A function's funExpr is its body only when the body is a function, and a bare RETURN, an abstract function's body, was taken for any function. A dynamic function's value is a closure of its funExpr, made as its class boots for a static one and as an instance is made for a member, so with a bare RETURN it was a closure of no function: the file loaded, and the next collection that marked a static one crashed, as did a call of a member one, interpreted and under the JIT. Haxe has no abstract dynamic function ("Invalid modifier combination"), so only a malformed file gives one. Such a file now fails to load with "Dynamic function is not a function". The check sits with the one for a body that is no function and no bare RETURN, so an interface's functions, which have no body, are left as they were. Tests: CppiaDynamicFunctionBody loads such files, then boots, runs and collects, interpreted and with the JIT. openfl/libhxcpp#257. --- Changes.md | 1 + src/hx/cppia/CppiaFunction.cpp | 4 ++ .../CppiaDynamicFunctionBody/Main.hx | 37 +++++++++++++++++++ .../CppiaDynamicFunctionBody/build.hxml | 4 ++ .../CppiaDynamicFunctionBody/stdout.txt | 10 +++++ 5 files changed, 56 insertions(+) create mode 100644 test/regression/CppiaDynamicFunctionBody/Main.hx create mode 100644 test/regression/CppiaDynamicFunctionBody/build.hxml create mode 100644 test/regression/CppiaDynamicFunctionBody/stdout.txt diff --git a/Changes.md b/Changes.md index f641bc037..b7c49e92a 100644 --- a/Changes.md +++ b/Changes.md @@ -16,6 +16,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed a malformed cppia file that calls the super constructor of a class with no constructor, in it or in a script class above it, crashing the loader, which read the constructor through a null pointer. It fails to load with "Super constructor call on a class with no constructor". The fuzz job found it (libhxcpp #257) * Fixed a malformed cppia file whose constructor was a bare return or, in an interface, had no body loading and then crashing: a super call into it crashed the loader, and a new of its class crashed when it ran, through the constructor's missing function. It fails to load with "Constructor is not a function" (libhxcpp #257) * Fixed a malformed cppia file that makes a new instance of a script interface loading and then crashing when it ran, interpreted and under the JIT: an interface has no host base to make the object from. It fails to load with "Could not link new" and the interface's name, as for a type with nothing to construct (libhxcpp #257) +* Fixed a malformed cppia file whose dynamic function was a bare return loading and then crashing, at the next collection after its class booted or when it was called, interpreted and under the JIT: its value was a closure of no function. It fails to load with "Dynamic function is not a function" (libhxcpp #257) libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/CppiaFunction.cpp b/src/hx/cppia/CppiaFunction.cpp index 9ab16903c..9e2f8eaeb 100644 --- a/src/hx/cppia/CppiaFunction.cpp +++ b/src/hx/cppia/CppiaFunction.cpp @@ -1176,6 +1176,10 @@ void CppiaFunction::load(CppiaStream &stream,bool inExpectBody) funExpr = dynamic_cast(body); if (!funExpr && name!="__init__" && std::string(body->getName())!="RetVal") throw "Function body is not a function"; + // A dynamic function's value is a closure of the function, made as its + // class boots or an instance is made, so it cannot be abstract + if (!funExpr && isDynamic) + throw "Dynamic function is not a function"; } stream.module->creatingFunction = 0; } diff --git a/test/regression/CppiaDynamicFunctionBody/Main.hx b/test/regression/CppiaDynamicFunctionBody/Main.hx new file mode 100644 index 000000000..93d506171 --- /dev/null +++ b/test/regression/CppiaDynamicFunctionBody/Main.hx @@ -0,0 +1,37 @@ +// A dynamic function's value is a closure of its function, made as its class +// boots or an instance is made. A malformed cppia file can give one a bare +// RETURN for its body, as an abstract function has, which is no function: the +// file loaded, and the closure of nothing crashed the next collection that +// marked it, or a call of it. Such a file fails to load now. +class Main { + // Strings "new" and "f"; class Bar with the field given, and a main + static function bar(field:String, main:String) { + return 'CPPIA 2 3 new 1 f 2 0 3 Bar 1 CLASS 1 0 0 1 $field MAIN 0 0 $main RESOURCES 0 '; + } + + static function check(name:String, code:String) { + try { + var module = cpp.cppia.Module.fromString(code); + module.boot(); + module.run(); + cpp.vm.Gc.run(true); + Sys.println('$name loaded and ran'); + untyped module.release(); + } catch (e:Dynamic) { + Sys.println('$name threw: $e'); + } + } + + static function main() { + final makeBar = "NEW 1 0"; + final callF = "CALLMEMBER 1 1 0 0 0 NEW 1 0"; + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + Sys.println(jit ? "jit" : "interpreted"); + check("static dynamic function that is a return", bar("FUNCTION 1 1 1 0 0 0 0 RETURN", makeBar)); + check("dynamic function that is a return", bar("FUNCTION 0 1 1 0 0 0 0 RETURN", callF)); + check("static dynamic function", bar("FUNCTION 1 1 1 0 0 0 0 FUN 0 0 0 0 BLOCK 0", makeBar)); + check("dynamic function", bar("FUNCTION 0 1 1 0 0 0 0 FUN 0 0 0 0 BLOCK 0", callF)); + } + } +} diff --git a/test/regression/CppiaDynamicFunctionBody/build.hxml b/test/regression/CppiaDynamicFunctionBody/build.hxml new file mode 100644 index 000000000..e497cddec --- /dev/null +++ b/test/regression/CppiaDynamicFunctionBody/build.hxml @@ -0,0 +1,4 @@ +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaDynamicFunctionBody/stdout.txt b/test/regression/CppiaDynamicFunctionBody/stdout.txt new file mode 100644 index 000000000..2aac600fc --- /dev/null +++ b/test/regression/CppiaDynamicFunctionBody/stdout.txt @@ -0,0 +1,10 @@ +interpreted +static dynamic function that is a return threw: Error reading file Dynamic function is not a function, line 1, char 75 +dynamic function that is a return threw: Error reading file Dynamic function is not a function, line 1, char 75 +static dynamic function loaded and ran +dynamic function loaded and ran +jit +static dynamic function that is a return threw: Error reading file Dynamic function is not a function, line 1, char 75 +dynamic function that is a return threw: Error reading file Dynamic function is not a function, line 1, char 75 +static dynamic function loaded and ran +dynamic function loaded and ran From 98f5dd5285434ef1e108e222d0dbec149c4e03aa Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:53:30 -0500 Subject: [PATCH 18/31] Hold a weak map's keys while a walk is on them A weak map's toString allocates as it walks the buckets, so a collection can run in the middle of the walk. That collection unlinked the entries whose keys had died, and the walk went on from its current entry's next, which still led to them when the current entry had been unlinked too: it called toString on keys the collector had finalized and freed, and on whatever had since taken their memory. 200 walks with a full collection part way through reached 54 finalized keys, and crashed once their memory was reused. While a walk is on a weak map, marking it marks its keys and values as a strong map does, so no entry is unlinked under the walk. Starting a walk remembers the map, so an incremental cycle that has already scanned it, or a minor collection of an old map, marks it again. The keys go at the first collection after the walk. --- .github/workflows/test.yml | 2 +- Changes.md | 1 + src/hx/Hash.h | 23 +++- test/regression/GcWeakMapWalkCollect/Main.hx | 109 ++++++++++++++++++ .../GcWeakMapWalkCollect/build.hxml | 2 + .../GcWeakMapWalkCollect/stdout.txt | 2 + 6 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 test/regression/GcWeakMapWalkCollect/Main.hx create mode 100644 test/regression/GcWeakMapWalkCollect/build.hxml create mode 100644 test/regression/GcWeakMapWalkCollect/stdout.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88d2cdb55..007a0b254 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -495,7 +495,7 @@ jobs: working-directory: test/regression run: | failed="" - for test in GcMutationRoots GcContainerMutation IncrementalGcSlices IncrementalGcContainers IncrementalGcArrayChunks IncrementalGcDequeMoves IncrementalGcPacing IncrementalGcStartSweep IncrementalGcStats IncrementalGcFinishBoundary IncrementalGcVerifier IncrementalWindowsHeaders GcWeakMapChain GenerationalFinalizerLifecycle GenerationalLargeBufferLifecycle; do + for test in GcMutationRoots GcContainerMutation IncrementalGcSlices IncrementalGcContainers IncrementalGcArrayChunks IncrementalGcDequeMoves IncrementalGcPacing IncrementalGcStartSweep IncrementalGcStats IncrementalGcFinishBoundary IncrementalGcVerifier IncrementalWindowsHeaders GcWeakMapChain GcWeakMapWalkCollect GenerationalFinalizerLifecycle GenerationalLargeBufferLifecycle; do echo "::group::incremental gc: $test" # The lifecycle fixtures' hxml selects generational mode; compile # the same Main with incremental mode here instead. diff --git a/Changes.md b/Changes.md index b7c49e92a..4d1c6f09a 100644 --- a/Changes.md +++ b/Changes.md @@ -17,6 +17,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed a malformed cppia file whose constructor was a bare return or, in an interface, had no body loading and then crashing: a super call into it crashed the loader, and a new of its class crashed when it ran, through the constructor's missing function. It fails to load with "Constructor is not a function" (libhxcpp #257) * Fixed a malformed cppia file that makes a new instance of a script interface loading and then crashing when it ran, interpreted and under the JIT: an interface has no host base to make the object from. It fails to load with "Could not link new" and the interface's name, as for a type with nothing to construct (libhxcpp #257) * Fixed a malformed cppia file whose dynamic function was a bare return loading and then crashing, at the next collection after its class booted or when it was called, interpreted and under the JIT: its value was a closure of no function. It fails to load with "Dynamic function is not a function" (libhxcpp #257) +* Fixed a weak map's toString calling toString on keys the collector had already freed, and on whatever had taken their memory, which could crash: a collection during the walk, which allocates as it goes, unlinked the entries whose keys had died, and the walk went on from its entry's next, which still led to them. A weak map now holds its keys and values while a walk is on them, and lets them go at the next collection after it libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/Hash.h b/src/hx/Hash.h index af57a2be1..b6ee9ee52 100644 --- a/src/hx/Hash.h +++ b/src/hx/Hash.h @@ -937,10 +937,21 @@ struct Hash : public HashBase< typename ELEMENT::Key > } + // A weak map holds its keys and values while a walk is on them. A + // collection the walk allocated for, as toString does, unlinked the + // entries whose keys died, and the walk went on from its entry's next, + // which still led to them, to keys already freed. The map is marked again + // where a collection might otherwise skip it: an old map in a minor + // collection, or one an incremental cycle has already scanned struct AutoWalk { Hash *hash; - AutoWalk(Hash *inHash) : hash(inHash) { hash->walks++; } + AutoWalk(Hash *inHash) : hash(inHash) + { + hash->walks++; + if (Element::WeakKeys) + HX_OBJ_WB_PESSIMISTIC_GET(hash); + } ~AutoWalk() { hash->walks--; } }; @@ -1104,14 +1115,16 @@ struct Hash : public HashBase< typename ELEMENT::Key > // Mark ... + // inStrong marks the keys and values of a weak map too: see AutoWalk struct HashMarker { hx::MarkContext *__inCtx; - HashMarker(hx::MarkContext *ctx) : __inCtx(ctx) { } + bool strong; + HashMarker(hx::MarkContext *ctx, bool inStrong) : __inCtx(ctx), strong(!Hash::Element::WeakKeys || inStrong) { } void operator()(typename Hash::Element *inElem) { HX_MARK_ARRAY(inElem); - if (!Hash::Element::WeakKeys) + if (strong) { HX_MARK_MEMBER(inElem->key); HX_MARK_MEMBER(inElem->value); @@ -1127,14 +1140,14 @@ struct Hash : public HashBase< typename ELEMENT::Key > if (size > 256 && hx::IncrementalRange(this,IncrementalScan,__inCtx)) return; #endif - HashMarker marker(__inCtx); + HashMarker marker(__inCtx, walks!=0); iterateBuckets(marker); } #ifdef HXCPP_GC_INCREMENTAL static bool IncrementalScan(hx::Object *inOwner, size_t &ioBucket, size_t &ioItem, hx::MarkContext *__inCtx) { Hash *hash = static_cast(inOwner); - HashMarker marker(__inCtx); + HashMarker marker(__inCtx, hash->walks!=0); Element *element = reinterpret_cast(ioItem); unsigned int work = 0; while (ioBucket < static_cast(hash->bucketCount) && work < 256) { diff --git a/test/regression/GcWeakMapWalkCollect/Main.hx b/test/regression/GcWeakMapWalkCollect/Main.hx new file mode 100644 index 000000000..1f2c9b107 --- /dev/null +++ b/test/regression/GcWeakMapWalkCollect/Main.hx @@ -0,0 +1,109 @@ +import cpp.vm.Gc; +import haxe.ds.WeakMap; + +class Key { + public var a:Int; + public var b:Int; + public var dead = false; + + public function new(i:Int) { + a = i; + b = i; + } + + public function toString() { + if (dead) + Main.deadReached++; + return "k"; + } +} + +// The size of a Key, to take the memory of a freed one +class Filler { + public var a = 0; + public var b = 0; + public var c = false; + + public function new() {} + + public function toString() { + Main.fillerReached++; + return "f"; + } +} + +class Val { + public function new() {} + + // Lets go of every key and collects, partway through the map's toString + public function toString() { + if (++Main.visits == Main.collectAt) { + for (i in 0...Main.held.length) + Main.held[i] = null; + Gc.run(true); + for (i in 0...20000) + Main.fillers.push(new Filler()); + } + return "v"; + } +} + +// A collection during a weak map's toString, which allocates as it goes, +// unlinked the entries whose keys had died, and the walk went on from its +// entry's next, which still led to them: it called toString on keys the +// collector had finalized and freed, and on whatever took their memory. A +// weak map now holds its keys and values while a walk is on them. +class Main { + public static var deadReached = 0; + public static var fillerReached = 0; + public static var visits = 0; + public static var collectAt = 0; + public static var held:Array = []; + public static var fillers:Array = []; + static var finalized = 0; + + static function onDead(k:Key) { + k.dead = true; + finalized++; + } + + static function fill(map:WeakMap, n:Int) { + held = []; + for (i in 0...n) { + var k = new Key(i); + Gc.setFinalizer(k, cpp.Callable.fromStaticFunction(onDead)); + held.push(k); + map.set(k, new Val()); + } + } + + static function walk(at:Int) { + var map = new WeakMap(); + fill(map, 64); + visits = 0; + collectAt = at; + map.toString(); + fillers = []; + } + + // Once the walk is over the keys go as before + static function keysGoAfterWalk():Bool { + var map = new WeakMap(); + fill(map, 64); + collectAt = -1; + map.toString(); + for (i in 0...held.length) + held[i] = null; + finalized = 0; + Gc.run(true); + Gc.run(true); + return finalized > 0; + } + + static function main() { + for (t in 0...200) + walk(1 + t % 32); + Sys.println('walk reached $deadReached dead keys and $fillerReached other objects'); + Sys.println('keys collected after the walk: ${keysGoAfterWalk()}'); + } +} diff --git a/test/regression/GcWeakMapWalkCollect/build.hxml b/test/regression/GcWeakMapWalkCollect/build.hxml new file mode 100644 index 000000000..76e908c18 --- /dev/null +++ b/test/regression/GcWeakMapWalkCollect/build.hxml @@ -0,0 +1,2 @@ +-m Main +-cpp bin diff --git a/test/regression/GcWeakMapWalkCollect/stdout.txt b/test/regression/GcWeakMapWalkCollect/stdout.txt new file mode 100644 index 000000000..a4fbfefc6 --- /dev/null +++ b/test/regression/GcWeakMapWalkCollect/stdout.txt @@ -0,0 +1,2 @@ +walk reached 0 dead keys and 0 other objects +keys collected after the walk: true From 2fb328f944015e10f95ba2b24e74f2fe8fe4a53f Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 07:25:34 -0500 Subject: [PATCH 19/31] Build the large allocation indexes on one marking thread at a time GetMemType sorts the bases of the large allocations into an index on the first lookups of a collection (#318), on the assumption that only the collecting thread looks up while marking. The marking threads look up too: a CFFI abstract made with create_abstract and memory of its own marks that memory conservatively in its __Mark, which any of them runs. Two threads could resize and sort one index at once, and one could search an index another was still filling, so a large buffer the abstract's memory pointed to went unmarked and was freed while held. With 20,000 buffers of 4,000 bytes held only through the memory of 8,000 abstracts, 10 to 14 of 16 full collections lost buffers, or the process crashed. With the marking on one thread (HXCPP_GC_VERIFY) none did. The index is built under a mutex, its ready flag is published with release and read with acquire, and the lookup counters are atomic. --- Changes.md | 2 + src/hx/gc/Immix.cpp | 64 ++++++++------ test/regression/GcAbstractLargeIndex/Main.hx | 83 +++++++++++++++++++ .../GcAbstractLargeIndex/build.hxml | 2 + .../GcAbstractLargeIndex/stdout.txt | 1 + 5 files changed, 127 insertions(+), 25 deletions(-) create mode 100644 test/regression/GcAbstractLargeIndex/Main.hx create mode 100644 test/regression/GcAbstractLargeIndex/build.hxml create mode 100644 test/regression/GcAbstractLargeIndex/stdout.txt diff --git a/Changes.md b/Changes.md index 4d1c6f09a..1f4168aac 100644 --- a/Changes.md +++ b/Changes.md @@ -17,6 +17,8 @@ libhxcpp 6.0.0 (unreleased) * Fixed a malformed cppia file whose constructor was a bare return or, in an interface, had no body loading and then crashing: a super call into it crashed the loader, and a new of its class crashed when it ran, through the constructor's missing function. It fails to load with "Constructor is not a function" (libhxcpp #257) * Fixed a malformed cppia file that makes a new instance of a script interface loading and then crashing when it ran, interpreted and under the JIT: an interface has no host base to make the object from. It fails to load with "Could not link new" and the interface's name, as for a type with nothing to construct (libhxcpp #257) * Fixed a malformed cppia file whose dynamic function was a bare return loading and then crashing, at the next collection after its class booted or when it was called, interpreted and under the JIT: its value was a closure of no function. It fails to load with "Dynamic function is not a function" (libhxcpp #257) +* Fixed the collector freeing large buffers that the memory of a CFFI abstract made with create_abstract pointed to, and crashing: the collector sorts the large allocations into an index on the first lookups of a collection, and the marking threads, which mark an abstract's memory conservatively, could build it two at once. One thread builds it now. In a test holding 20,000 buffers through 8,000 abstracts, 10 to 14 collections in 16 lost buffers or the process crashed + * Fixed a weak map's toString calling toString on keys the collector had already freed, and on whatever had taken their memory, which could crash: a collection during the walk, which allocates as it goes, unlinked the entries whose keys had died, and the walk went on from its entry's next, which still led to them. A weak map now holds its keys and values while a walk is on them, and lets them go at the next collection after it libhxcpp 6.0.0-rc.2 diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index 28d3768ba..e9edacf3c 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -7621,21 +7621,28 @@ class GlobalAllocator return memUnmanaged; uintptr_t base = 0; - // Only the collector looks up while marking, with the list fixed, so once - // a collection has seen several candidates in range it sorts the bases - // and searches them instead of scanning the list for each. The old - // allocations' sorted bases are kept while they do not change, which a - // minor collection cannot do: sorting them again in each collection - // took longer than sweeping them did (libhxcpp #318). - if (!mLargeIndexReady && mLargeOld>64 && ++mLargeLookups>8) - { - mLargeIndex.resize(mLargeOld); - for(int i=0;i(mLargeList[i] + 2); - std::sort(mLargeIndex.begin(), mLargeIndex.end()); - mLargeIndexReady = true; + // The list is fixed while marking, so once a collection has seen several + // candidates in range it sorts the bases and searches them instead of + // scanning the list for each. The old allocations' sorted bases are kept + // while they do not change, which a minor collection cannot do: sorting + // them again in each collection took longer than sweeping them did + // (libhxcpp #318). The marking threads look up too, for a CFFI abstract + // whose memory is marked conservatively, so one thread sorts, under + // mLargeIndexLock: two sorting one index at once crashed + if (!mLargeIndexReady.load(std::memory_order_acquire) && mLargeOld>64 && + mLargeLookups.fetch_add(1, std::memory_order_relaxed)>=8) + { + std::lock_guard lock(mLargeIndexLock); + if (!mLargeIndexReady.load(std::memory_order_relaxed)) + { + mLargeIndex.resize(mLargeOld); + for(int i=0;i(mLargeList[i] + 2); + std::sort(mLargeIndex.begin(), mLargeIndex.end()); + mLargeIndexReady.store(true, std::memory_order_release); + } } - if (mLargeIndexReady) + if (mLargeIndexReady.load(std::memory_order_acquire)) base = FindLarge(mLargeIndex, inPtr, reach); else for(int i=0;i64 && ++mYoungLargeLookups>8) + if (!mYoungLargeIndexReady.load(std::memory_order_acquire) && young>64 && + mYoungLargeLookups.fetch_add(1, std::memory_order_relaxed)>=8) { - mYoungLargeIndex.resize(young); - for(int i=0;i(mLargeList[mLargeOld+i] + 2); - std::sort(mYoungLargeIndex.begin(), mYoungLargeIndex.end()); - mYoungLargeIndexReady = true; + std::lock_guard lock(mLargeIndexLock); + if (!mYoungLargeIndexReady.load(std::memory_order_relaxed)) + { + mYoungLargeIndex.resize(young); + for(int i=0;i(mLargeList[mLargeOld+i] + 2); + std::sort(mYoungLargeIndex.begin(), mYoungLargeIndex.end()); + mYoungLargeIndexReady.store(true, std::memory_order_release); + } } - if (mYoungLargeIndexReady) + if (mYoungLargeIndexReady.load(std::memory_order_acquire)) base = FindLarge(mYoungLargeIndex, inPtr, reach); else for(int i=mLargeOld;i mLargeIndex; - bool mLargeIndexReady = false; - int mLargeLookups = 0; + std::atomic mLargeIndexReady{false}; + std::atomic mLargeLookups{0}; // The new survivors' bases, taken for the index before a full sweep std::vector mLargeIndexNew; std::vector mYoungLargeIndex; - bool mYoungLargeIndexReady = false; - int mYoungLargeLookups = 0; + std::atomic mYoungLargeIndexReady{false}; + std::atomic mYoungLargeLookups{0}; + std::mutex mLargeIndexLock; size_t mTotalAfterLastCollect; size_t mAllBlocksCount; size_t mLiveBytes; diff --git a/test/regression/GcAbstractLargeIndex/Main.hx b/test/regression/GcAbstractLargeIndex/Main.hx new file mode 100644 index 000000000..5c14aefea --- /dev/null +++ b/test/regression/GcAbstractLargeIndex/Main.hx @@ -0,0 +1,83 @@ +import cpp.vm.Gc; +import haxe.io.Bytes; + +@:cppFileCode(' +extern "C" void *hx_cffi(const char *inName); +typedef hx::Object *(*CreateAbstract)(void *, int, void *); + +// A CFFI abstract with memory of its own, which the collector marks +// conservatively, as it does a stack +static Dynamic makeAbstract(int inWords) +{ + static CreateAbstract create = (CreateAbstract)hx_cffi("create_abstract"); + return create((void *)(intptr_t)0x1234, inWords*(int)sizeof(void *), 0); +} + +static void setWord(Dynamic inAbstract, int inIndex, void *inValue) +{ + ((void **)inAbstract->__GetHandle())[inIndex] = inValue; +} +') +// The collector finds a large allocation from a word that may point at one +// through sorted indexes of the large allocations, which it builds on the +// first lookups of a collection. The marking threads look up too, for an +// abstract's memory, and two of them building one index at once left +// buffers that the abstracts' memory pointed to unmarked: they were freed +// while held, and taken again by the next allocations. +class Main { + static inline var BUFFERS = 20000; + static inline var BUFFER_SIZE = 4000; + static inline var ABSTRACTS = 8000; + static inline var WORDS = 16; + + static var keep:Array; + static var bases:Array>; + + // The buffers are held only through the abstracts' memory once this returns + static function round(r:Int) { + bases = []; + var hold = []; + for (j in 0...BUFFERS) { + var b = Bytes.alloc(BUFFER_SIZE); + hold.push(b); + b.fill(0, BUFFER_SIZE, (r * 7 + j) & 255); + bases.push(cpp.Pointer.arrayElem(b.getData(), 0)); + } + keep = []; + for (a in 0...ABSTRACTS) { + var abs:Dynamic = untyped __cpp__("makeAbstract({0})", WORDS); + for (w in 0...WORDS) { + var p = bases[(a * WORDS + w) % BUFFERS]; + untyped __cpp__("setWord({0}, {1}, (void *){2}.ptr)", abs, w, p); + } + keep.push(abs); + } + for (j in 0...BUFFERS) + hold[j] = null; + } + + static function intact(r:Int):Bool { + for (j in 0...BUFFERS) { + var p = bases[j]; + var want = (r * 7 + j) & 255; + for (k in 0...BUFFER_SIZE) + if (p[k] != want) + return false; + } + return true; + } + + static function main() { + var lost = 0; + for (r in 0...16) { + round(r); + Gc.run(true); + // What the collection freed is taken again here + for (j in 0...BUFFERS) + Bytes.alloc(BUFFER_SIZE).fill(0, BUFFER_SIZE, 0xee); + if (!intact(r)) + lost++; + } + Sys.println('rounds that lost a buffer an abstract held: $lost'); + } +} diff --git a/test/regression/GcAbstractLargeIndex/build.hxml b/test/regression/GcAbstractLargeIndex/build.hxml new file mode 100644 index 000000000..76e908c18 --- /dev/null +++ b/test/regression/GcAbstractLargeIndex/build.hxml @@ -0,0 +1,2 @@ +-m Main +-cpp bin diff --git a/test/regression/GcAbstractLargeIndex/stdout.txt b/test/regression/GcAbstractLargeIndex/stdout.txt new file mode 100644 index 000000000..11314e649 --- /dev/null +++ b/test/regression/GcAbstractLargeIndex/stdout.txt @@ -0,0 +1 @@ +rounds that lost a buffer an abstract held: 0 From 8ec29c921ae9e4464773c7c0ac9efa28b27d88a5 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 07:28:19 -0500 Subject: [PATCH 20/31] Read a large nursery allocation's size past the flags in its header With HXCPP_GC_NURSERY a small allocation keeps its size in the low 16 bits of its header and a large one 0 there. ObjectSizeSafe took any bit of the low 24 as a small size, so a large allocation with a flag, as a string of 4,000 bytes or more has for its hash slot or its UTF-16 text, gave a size of 0. _hx_gc_freeze copied none of such a string, and the frozen string read past its copy. ObjectSizeSafe reads the 16 size bits, as MarkAllocUnchecked already did. --- Changes.md | 2 ++ src/hx/gc/Immix.cpp | 7 ++-- test/regression/GcFreezeNurseryLarge/Main.hx | 33 +++++++++++++++++++ .../GcFreezeNurseryLarge/build.hxml | 3 ++ .../GcFreezeNurseryLarge/stdout.txt | 3 ++ 5 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 test/regression/GcFreezeNurseryLarge/Main.hx create mode 100644 test/regression/GcFreezeNurseryLarge/build.hxml create mode 100644 test/regression/GcFreezeNurseryLarge/stdout.txt diff --git a/Changes.md b/Changes.md index 1f4168aac..ef0113d6d 100644 --- a/Changes.md +++ b/Changes.md @@ -17,6 +17,8 @@ libhxcpp 6.0.0 (unreleased) * Fixed a malformed cppia file whose constructor was a bare return or, in an interface, had no body loading and then crashing: a super call into it crashed the loader, and a new of its class crashed when it ran, through the constructor's missing function. It fails to load with "Constructor is not a function" (libhxcpp #257) * Fixed a malformed cppia file that makes a new instance of a script interface loading and then crashing when it ran, interpreted and under the JIT: an interface has no host base to make the object from. It fails to load with "Could not link new" and the interface's name, as for a type with nothing to construct (libhxcpp #257) * Fixed a malformed cppia file whose dynamic function was a bare return loading and then crashing, at the next collection after its class booted or when it was called, interpreted and under the JIT: its value was a closure of no function. It fails to load with "Dynamic function is not a function" (libhxcpp #257) +* Fixed _hx_gc_freeze copying none of a string of 4,000 bytes or more with HXCPP_GC_NURSERY, so the frozen string read past its copy: the size of a large allocation in the nursery was taken as 0 when its header had a flag, as a long string's hash slot or UTF-16 text sets + * Fixed the collector freeing large buffers that the memory of a CFFI abstract made with create_abstract pointed to, and crashing: the collector sorts the large allocations into an index on the first lookups of a collection, and the marking threads, which mark an abstract's memory conservatively, could build it two at once. One thread builds it now. In a test holding 20,000 buffers through 8,000 abstracts, 10 to 14 collections in 16 lost buffers or the process crashed * Fixed a weak map's toString calling toString on keys the collector had already freed, and on whatever had taken their memory, which could crash: a collection during the walk, which allocates as it goes, unlinked the entries whose keys had died, and the walk went on from its entry's next, which still led to them. A weak map now holds its keys and values while a walk is on them, and lets them go at the next collection after it diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index e9edacf3c..6d9159b56 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -9201,8 +9201,11 @@ unsigned int ObjectSizeSafe(void *inData) #ifdef HXCPP_GC_NURSERY if (!(header & 0xff000000)) { - // Small object - if (header & 0x00ffffff) + // A small object has its size in the low 16 bits, and a large one 0 + // there, as MarkAllocUnchecked reads it. The flags above can be set + // on either: a long string's hash slot made a large one look small, + // with a size of 0 + if (header & 0x0000ffff) return header & 0x0000ffff; // Large object } diff --git a/test/regression/GcFreezeNurseryLarge/Main.hx b/test/regression/GcFreezeNurseryLarge/Main.hx new file mode 100644 index 000000000..5b3e77e31 --- /dev/null +++ b/test/regression/GcFreezeNurseryLarge/Main.hx @@ -0,0 +1,33 @@ +class Holder { + public var text:String; + public var wide:String; + + public function new(t:String, w:String) { + text = t; + wide = w; + } +} + +// With HXCPP_GC_NURSERY, ObjectSizeSafe took a large allocation with a flag +// in its header, as a long string has for its hash slot or its UTF-16 data, +// for a small one, and gave its size as 0: _hx_gc_freeze copied none of it, +// and the frozen string read past its copy. +class Main { + static function freeze(value:T):T + return untyped __cpp__("_hx_gc_freeze({0})", value); + + static function size(s:String):Int + return untyped __cpp__("(int)::hx::ObjectSizeSafe((void *){0}.raw_ptr())", s); + + static function main() { + var buf = new StringBuf(); + for (i in 0...5000) + buf.add(String.fromCharCode(97 + i % 26)); + var text = buf.toString(); + var wide = text + "é"; + var frozen = freeze(new Holder(text, wide)); + Sys.println('size covers the string: ${size(text) >= 5000}'); + Sys.println('frozen copy equal: ${frozen.text == text}'); + Sys.println('frozen UTF-16 copy equal: ${frozen.wide == wide}'); + } +} diff --git a/test/regression/GcFreezeNurseryLarge/build.hxml b/test/regression/GcFreezeNurseryLarge/build.hxml new file mode 100644 index 000000000..e7b583ab2 --- /dev/null +++ b/test/regression/GcFreezeNurseryLarge/build.hxml @@ -0,0 +1,3 @@ +-m Main +--cpp bin +-D HXCPP_GC_NURSERY diff --git a/test/regression/GcFreezeNurseryLarge/stdout.txt b/test/regression/GcFreezeNurseryLarge/stdout.txt new file mode 100644 index 000000000..6f1724ab8 --- /dev/null +++ b/test/regression/GcFreezeNurseryLarge/stdout.txt @@ -0,0 +1,3 @@ +size covers the string: true +frozen copy equal: true +frozen UTF-16 copy equal: true From b67222a13cb59dc645ba6ebc88d416b206c35a00 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:45:42 -0500 Subject: [PATCH 21/31] Wrap Int array *= and -= under the cppia JIT The JIT's compound assignment to an array element gave * and - a Float right-hand side for every element type, so on an Array it worked in Float and converted the result back to Int. A result past the Int range converted to -2147483648, where compiled code and the interpreter wrap: values[i] *= 3 with 2147483647 gave -2147483648 instead of 2147483645, and a hash kept in an array went wrong as soon as it overflowed. An Int element now takes an Int right-hand side and Int arithmetic. The new test runs under the interpreter and the JIT. --- Changes.md | 1 + src/hx/cppia/ArrayBuiltin.cpp | 8 +++- src/hx/cppia/Cppia.h | 3 +- .../regression/CppiaJitIntArrayAssign/Main.hx | 11 ++++++ .../CppiaJitIntArrayAssign/Script.hx | 37 +++++++++++++++++++ .../CppiaJitIntArrayAssign/build.hxml | 7 ++++ .../CppiaJitIntArrayAssign/stdout.txt | 8 ++++ 7 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 test/regression/CppiaJitIntArrayAssign/Main.hx create mode 100644 test/regression/CppiaJitIntArrayAssign/Script.hx create mode 100644 test/regression/CppiaJitIntArrayAssign/build.hxml create mode 100644 test/regression/CppiaJitIntArrayAssign/stdout.txt diff --git a/Changes.md b/Changes.md index ef0113d6d..0e5637baa 100644 --- a/Changes.md +++ b/Changes.md @@ -22,6 +22,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed the collector freeing large buffers that the memory of a CFFI abstract made with create_abstract pointed to, and crashing: the collector sorts the large allocations into an index on the first lookups of a collection, and the marking threads, which mark an abstract's memory conservatively, could build it two at once. One thread builds it now. In a test holding 20,000 buffers through 8,000 abstracts, 10 to 14 collections in 16 lost buffers or the process crashed * Fixed a weak map's toString calling toString on keys the collector had already freed, and on whatever had taken their memory, which could crash: a collection during the walk, which allocates as it goes, unlinked the entries whose keys had died, and the walk went on from its entry's next, which still led to them. A weak map now holds its keys and values while a walk is on them, and lets them go at the next collection after it +* Fixed *= and -= on an element of an Array under the cppia JIT giving -2147483648 when the result passed the Int range: the JIT did them in Float arithmetic and converted back, where compiled code and the interpreter wrap, so a hash kept in an array, h[i] *= 31, went wrong once it overflowed libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/ArrayBuiltin.cpp b/src/hx/cppia/ArrayBuiltin.cpp index 6ecf7e99a..27587af64 100644 --- a/src/hx/cppia/ArrayBuiltin.cpp +++ b/src/hx/cppia/ArrayBuiltin.cpp @@ -269,8 +269,14 @@ struct ArraySetter : public ArrayBuiltinBase switch((int)FUNC::op) { case aoMult: - case aoDiv: case aoSub: + // Int elements stay in Int arithmetic, which wraps as compiled + // code does: through a Float, a result past the Int range + // converted to -2147483648 + rightHandType = elemType == etInt ? etInt : etFloat; + break; + + case aoDiv: case aoMod: rightHandType = etFloat; break; diff --git a/src/hx/cppia/Cppia.h b/src/hx/cppia/Cppia.h index bbff87958..c38c5d46e 100644 --- a/src/hx/cppia/Cppia.h +++ b/src/hx/cppia/Cppia.h @@ -1256,8 +1256,9 @@ struct NAME \ void apply( T &ioVal, double val) { AssignDouble(ioVal, ValToFloat(ioVal) OP val); } \ template static \ void apply( T &ioVal, String &val) { } \ + /* The jit's Int array *= and -=, never /= */ \ template static \ - void apply( T &ioVal, int &val) { } \ + void apply( T &ioVal, int &val) { AssignInt(ioVal, ValToInt(ioVal) OP val); } \ template static \ void apply( T &ioVal, Dynamic &val) { } \ }; diff --git a/test/regression/CppiaJitIntArrayAssign/Main.hx b/test/regression/CppiaJitIntArrayAssign/Main.hx new file mode 100644 index 000000000..24664e8bb --- /dev/null +++ b/test/regression/CppiaJitIntArrayAssign/Main.hx @@ -0,0 +1,11 @@ +function main() { + var script = haxe.io.Path.join([haxe.io.Path.directory(Sys.programPath()), "script.cppia"]); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + try { + cpp.cppia.Host.runFile(script); + } catch (e:Dynamic) { + Sys.println("error: " + e); + } + } +} diff --git a/test/regression/CppiaJitIntArrayAssign/Script.hx b/test/regression/CppiaJitIntArrayAssign/Script.hx new file mode 100644 index 000000000..8984bb138 --- /dev/null +++ b/test/regression/CppiaJitIntArrayAssign/Script.hx @@ -0,0 +1,37 @@ +class Hasher { + public var parts = [17, 17]; + + public function new() {} + + public function add(c:Int) { + parts[0] *= 31; + parts[0] += c; + parts[1] -= c * 1000003; + } +} + +// Int array *= and -= under the JIT went through Float arithmetic, so a +// result past the Int range converted to -2147483648 instead of wrapping as +// compiled code and the interpreter do +class Script { + static var values = [2147483647, -2147483648]; + + public static function main() { + var h = new Hasher(); + for (c in "hash me please".split("")) + h.add(c.charCodeAt(0)); + Sys.println(h.parts); + + var a = [0, 2147483647]; + var i = 1; + var r = (a[i] *= 3); + Sys.println(r + " " + a[i]); + a = [-2147483648]; + a[0] -= 3; + Sys.println(a[0]); + + values[0] *= 2; + values[1] -= 1; + Sys.println(values); + } +} diff --git a/test/regression/CppiaJitIntArrayAssign/build.hxml b/test/regression/CppiaJitIntArrayAssign/build.hxml new file mode 100644 index 000000000..d20d36bd4 --- /dev/null +++ b/test/regression/CppiaJitIntArrayAssign/build.hxml @@ -0,0 +1,7 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaJitIntArrayAssign/stdout.txt b/test/regression/CppiaJitIntArrayAssign/stdout.txt new file mode 100644 index 000000000..1907c2f9d --- /dev/null +++ b/test/regression/CppiaJitIntArrayAssign/stdout.txt @@ -0,0 +1,8 @@ +[322366769,-1328003967] +2147483645 2147483645 +2147483645 +[-2,2147483647] +[322366769,-1328003967] +2147483645 2147483645 +2147483645 +[-2,2147483647] From efe3d7ce0fa6777e650b329070cf25aa3f9b1bff Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:48:44 -0500 Subject: [PATCH 22/31] Keep a negated Int an Int in cppia The interpreter's negation gave a Float and a Dynamic made from -x whatever x was, and the JIT negated in Float arithmetic for every destination but an Int. So wherever -x was not used as an Int, as Dynamic, in a String or as a Float, an Int was negated as a Float: -0 printed as "-0", -(-2147483648) gave 2147483648 instead of wrapping as compiled code does, and Type.typeof gave TFloat. An Int is now negated as an Int, then converted. The new test runs under the interpreter and the JIT. --- Changes.md | 1 + src/hx/cppia/Cppia.cpp | 12 +++++++++++ test/regression/CppiaIntNegation/Main.hx | 11 +++++++++++ test/regression/CppiaIntNegation/Script.hx | 22 +++++++++++++++++++++ test/regression/CppiaIntNegation/build.hxml | 7 +++++++ test/regression/CppiaIntNegation/stdout.txt | 8 ++++++++ 6 files changed, 61 insertions(+) create mode 100644 test/regression/CppiaIntNegation/Main.hx create mode 100644 test/regression/CppiaIntNegation/Script.hx create mode 100644 test/regression/CppiaIntNegation/build.hxml create mode 100644 test/regression/CppiaIntNegation/stdout.txt diff --git a/Changes.md b/Changes.md index 0e5637baa..a7d509482 100644 --- a/Changes.md +++ b/Changes.md @@ -23,6 +23,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed a weak map's toString calling toString on keys the collector had already freed, and on whatever had taken their memory, which could crash: a collection during the walk, which allocates as it goes, unlinked the entries whose keys had died, and the walk went on from its entry's next, which still led to them. A weak map now holds its keys and values while a walk is on them, and lets them go at the next collection after it * Fixed *= and -= on an element of an Array under the cppia JIT giving -2147483648 when the result passed the Int range: the JIT did them in Float arithmetic and converted back, where compiled code and the interpreter wrap, so a hash kept in an array, h[i] *= 31, went wrong once it overflowed +* Fixed cppia negating an Int as a Float wherever the result was not used as an Int, interpreted and under the JIT: -x with x = 0 printed as "-0", -(-2147483648) gave 2147483648 instead of wrapping, and Type.typeof gave TFloat, as Dynamic, in a String or as a Float libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index b08789f11..f8b956fef 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -8015,12 +8015,18 @@ struct OpNeg : public CppiaExpr { return - value->runInt(ctx); } + // An Int negated stays an Int: -0 is 0 and -(-2147483648) wraps, as in + // compiled code Float runFloat(CppiaCtx *ctx) HXCPP_OVERRIDE { + if (type==etInt) + return runInt(ctx); return - value->runFloat(ctx); } hx::Object *runObject(CppiaCtx *ctx) HXCPP_OVERRIDE { + if (type==etInt) + return Dynamic(runInt(ctx)).mPtr; return Dynamic(- value->runFloat(ctx)).mPtr; } @@ -8032,6 +8038,12 @@ struct OpNeg : public CppiaExpr value->genCode(compiler, sJitTemp0.as(jtInt), etInt); compiler->negate(inDest, sJitTemp0.as(jtInt) ); } + else if (type==etInt) + { + value->genCode(compiler, sJitTemp0.as(jtInt), etInt); + compiler->negate(sJitTemp0.as(jtInt), sJitTemp0.as(jtInt) ); + compiler->convert(sJitTemp0, etInt, inDest, destType); + } else { value->genCode(compiler, sJitTempF0, etFloat); diff --git a/test/regression/CppiaIntNegation/Main.hx b/test/regression/CppiaIntNegation/Main.hx new file mode 100644 index 000000000..24664e8bb --- /dev/null +++ b/test/regression/CppiaIntNegation/Main.hx @@ -0,0 +1,11 @@ +function main() { + var script = haxe.io.Path.join([haxe.io.Path.directory(Sys.programPath()), "script.cppia"]); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + try { + cpp.cppia.Host.runFile(script); + } catch (e:Dynamic) { + Sys.println("error: " + e); + } + } +} diff --git a/test/regression/CppiaIntNegation/Script.hx b/test/regression/CppiaIntNegation/Script.hx new file mode 100644 index 000000000..ae6057d36 --- /dev/null +++ b/test/regression/CppiaIntNegation/Script.hx @@ -0,0 +1,22 @@ +// cppia negated an Int as a Float wherever the result was not an Int: as +// Dynamic, in a String or as a Float. -0 printed as "-0", +// -(-2147483648) gave 2147483648 and Type.typeof gave TFloat, interpreted +// and under the JIT +class Script { + static var zero = 0; + static var five = 5; + static var min = -2147483648; + + static function describe(d:Dynamic):String + return Std.string(d) + " " + Type.typeof(d); + + public static function main() { + Sys.println(describe(-zero) + ", " + describe(-five) + ", " + describe(-min)); + var asDynamic:Dynamic = -zero; + var row:Array = [-zero, -min]; + Sys.println(describe(asDynamic) + ", " + row); + Sys.println("" + (-zero) + " " + Std.string(-five) + " " + (-min)); + var f:Float = -min; + Sys.println(f); + } +} diff --git a/test/regression/CppiaIntNegation/build.hxml b/test/regression/CppiaIntNegation/build.hxml new file mode 100644 index 000000000..d20d36bd4 --- /dev/null +++ b/test/regression/CppiaIntNegation/build.hxml @@ -0,0 +1,7 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaIntNegation/stdout.txt b/test/regression/CppiaIntNegation/stdout.txt new file mode 100644 index 000000000..f13e8b9a2 --- /dev/null +++ b/test/regression/CppiaIntNegation/stdout.txt @@ -0,0 +1,8 @@ +0 TInt, -5 TInt, -2147483648 TInt +0 TInt, [0,-2147483648] +0 -5 -2147483648 +-2147483648 +0 TInt, -5 TInt, -2147483648 TInt +0 TInt, [0,-2147483648] +0 -5 -2147483648 +-2147483648 From f2a5530547523baa04f43065d6d430a219599577 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 00:48:53 -0500 Subject: [PATCH 23/31] Box Array contains and remove as Bool in interpreted cppia Array has builtins of its own, and their runObject boxed the results of contains and remove with the Int results of push and indexOf, before the case meant to box remove as a Bool, which was never reached. So in the interpreter "" + a.contains(x) was "1" and Std.isOfType(a.remove(x), Bool) was false; the JIT already boxed both as Bool. CppiaBoolBuiltins checks them too, under the interpreter and the JIT. --- Changes.md | 1 + src/hx/cppia/ArrayVirtual.cpp | 8 ++++---- test/regression/CppiaBoolBuiltins/Script.hx | 5 +++++ test/regression/CppiaBoolBuiltins/stdout.txt | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Changes.md b/Changes.md index a7d509482..b678ecbb4 100644 --- a/Changes.md +++ b/Changes.md @@ -24,6 +24,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed a weak map's toString calling toString on keys the collector had already freed, and on whatever had taken their memory, which could crash: a collection during the walk, which allocates as it goes, unlinked the entries whose keys had died, and the walk went on from its entry's next, which still led to them. A weak map now holds its keys and values while a walk is on them, and lets them go at the next collection after it * Fixed *= and -= on an element of an Array under the cppia JIT giving -2147483648 when the result passed the Int range: the JIT did them in Float arithmetic and converted back, where compiled code and the interpreter wrap, so a hash kept in an array, h[i] *= 31, went wrong once it overflowed * Fixed cppia negating an Int as a Float wherever the result was not used as an Int, interpreted and under the JIT: -x with x = 0 printed as "-0", -(-2147483648) gave 2147483648 instead of wrapping, and Type.typeof gave TFloat, as Dynamic, in a String or as a Float +* Fixed interpreted cppia giving contains and remove on an Array as an Int where they became Dynamic or a String: "" + a.contains(x) was "1" and Std.isOfType(a.remove(x), Bool) false. The fix for the typed arrays had missed Array's builtins libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/ArrayVirtual.cpp b/src/hx/cppia/ArrayVirtual.cpp index 04500f961..80812057e 100644 --- a/src/hx/cppia/ArrayVirtual.cpp +++ b/src/hx/cppia/ArrayVirtual.cpp @@ -260,12 +260,12 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase hx::Object *runObject(CppiaCtx *ctx) HXCPP_OVERRIDE { - if (FUNC==afPush || FUNC==afContains || FUNC==afRemove || FUNC==afIndexOf || FUNC==afLastIndexOf) - return Dynamic(runInt(ctx)).mPtr; - - if (FUNC==afRemove) + if (FUNC==afContains || FUNC==afRemove) return Dynamic((bool)runInt(ctx)).mPtr; + if (FUNC==afPush || FUNC==afIndexOf || FUNC==afLastIndexOf) + return Dynamic(runInt(ctx)).mPtr; + if (FUNC==afJoin || FUNC==afToString) return Dynamic(runString(ctx)).mPtr; diff --git a/test/regression/CppiaBoolBuiltins/Script.hx b/test/regression/CppiaBoolBuiltins/Script.hx index a60b77b27..0a361f25a 100644 --- a/test/regression/CppiaBoolBuiltins/Script.hx +++ b/test/regression/CppiaBoolBuiltins/Script.hx @@ -12,5 +12,10 @@ class Script { // In an Array literal the jit boxed these as Int var row:Array = [a.contains(1), a.remove(3), flags[0], flags[1] = false]; Sys.println(row + " " + [for (x in row) Std.isOfType(x, Bool)]); + // Array has builtins of its own, and the interpreter boxed these as Int + var any:Array = [1, "a", true]; + var ac:Dynamic = any.contains("a"); + var ar:Dynamic = any.remove(true); + Sys.println(Std.isOfType(ac, Bool) + " " + ac + " " + Std.isOfType(ar, Bool) + " " + ar + " " + any.contains(2)); } } diff --git a/test/regression/CppiaBoolBuiltins/stdout.txt b/test/regression/CppiaBoolBuiltins/stdout.txt index 786ebea31..dd44f81e7 100644 --- a/test/regression/CppiaBoolBuiltins/stdout.txt +++ b/test/regression/CppiaBoolBuiltins/stdout.txt @@ -3,8 +3,10 @@ true true true true true [true,true,true,false] [true,true,true,true] +true true true true false true true true true true true [true,true,true,false] [true,true,true,true] +true true true true false From 82f19f124fc8b135c71f25cf58791ca0cbc4fcb4 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 07:28:49 -0500 Subject: [PATCH 24/31] Bridge exceptions from concat under the cppia JIT concat with a null array throws Null Object Reference under HXCPP_CHECK_POINTER, and HXCPP_CATCH_SEGV, which the Cppia host is built with, turns the fault into an exception too. The JIT's native calls for concat, on typed arrays and on Array, had no exception bridge, so the exception went through jitted frames and ended the process, where the interpreter reaches the script's catch. They now store it as the script's pending exception, which the code after the call checks, as blit, setSize and resize do. The new test runs under the interpreter and the JIT, with HXCPP_CHECK_POINTER. --- Changes.md | 1 + src/hx/cppia/ArrayBuiltin.cpp | 6 ++++ src/hx/cppia/ArrayVirtual.cpp | 4 +++ test/regression/CppiaJitConcatNull/Main.hx | 13 ++++++++ test/regression/CppiaJitConcatNull/Script.hx | 30 +++++++++++++++++++ test/regression/CppiaJitConcatNull/build.hxml | 8 +++++ test/regression/CppiaJitConcatNull/stdout.txt | 10 +++++++ 7 files changed, 72 insertions(+) create mode 100644 test/regression/CppiaJitConcatNull/Main.hx create mode 100644 test/regression/CppiaJitConcatNull/Script.hx create mode 100644 test/regression/CppiaJitConcatNull/build.hxml create mode 100644 test/regression/CppiaJitConcatNull/stdout.txt diff --git a/Changes.md b/Changes.md index b678ecbb4..67372ba31 100644 --- a/Changes.md +++ b/Changes.md @@ -25,6 +25,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed *= and -= on an element of an Array under the cppia JIT giving -2147483648 when the result passed the Int range: the JIT did them in Float arithmetic and converted back, where compiled code and the interpreter wrap, so a hash kept in an array, h[i] *= 31, went wrong once it overflowed * Fixed cppia negating an Int as a Float wherever the result was not used as an Int, interpreted and under the JIT: -x with x = 0 printed as "-0", -(-2147483648) gave 2147483648 instead of wrapping, and Type.typeof gave TFloat, as Dynamic, in a String or as a Float * Fixed interpreted cppia giving contains and remove on an Array as an Int where they became Dynamic or a String: "" + a.contains(x) was "1" and Std.isOfType(a.remove(x), Bool) false. The fix for the typed arrays had missed Array's builtins +* Fixed concat with a null array under the cppia JIT ending the process where it throws Null Object Reference (with HXCPP_CHECK_POINTER, or HXCPP_CATCH_SEGV as the Cppia host has), on typed arrays and Array: the native call had no exception bridge. The script catches it, as it does interpreted libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/ArrayBuiltin.cpp b/src/hx/cppia/ArrayBuiltin.cpp index 27587af64..0ecacbf92 100644 --- a/src/hx/cppia/ArrayBuiltin.cpp +++ b/src/hx/cppia/ArrayBuiltin.cpp @@ -1266,7 +1266,12 @@ struct ArrayBuiltin : public ArrayBuiltinBase static hx::Object *SLJIT_CALL runConcat( Array_obj *inArray, Array_obj *inOther ) { + // A null array to append throws: the exception becomes the script's + // pending one, which the code after the call checks + TRY_NATIVE return inArray->concat(inOther).mPtr; + CATCH_NATIVE + return 0; } @@ -1897,6 +1902,7 @@ struct ArrayBuiltin : public ArrayBuiltinBase genNullReferenceExceptionCheck(compiler, thisVal); args[0]->genCode(compiler, sJitArg1.as(jtPointer), etObject); compiler->callNative( (void *)runConcat, thisVal, sJitArg1.as(jtPointer)); + compiler->checkException(); compiler->convertReturnReg(etObject, inDest, destType); break; } diff --git a/src/hx/cppia/ArrayVirtual.cpp b/src/hx/cppia/ArrayVirtual.cpp index 80812057e..46ba3fb6f 100644 --- a/src/hx/cppia/ArrayVirtual.cpp +++ b/src/hx/cppia/ArrayVirtual.cpp @@ -24,7 +24,10 @@ static int SLJIT_CALL arrayRemove(ArrayAnyImpl *inObj, hx::Object *inValue) } static hx::Object * SLJIT_CALL arrayConcat(ArrayAnyImpl *inObj, hx::Object *inValue) { + TRY_NATIVE return inObj->concat(Dynamic(inValue)).mPtr; + CATCH_NATIVE + return 0; } // These throw for a size or range the array cannot take: the exception becomes // the script's pending one, which the code after the call checks @@ -782,6 +785,7 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase { args[0]->genCode(compiler, sJitArg1, etObject); compiler->callNative( (void *)arrayConcat, thisVal, sJitArg1.as(jtPointer)); + compiler->checkException(); compiler->convertReturnReg(etObject, inDest, destType); } break; diff --git a/test/regression/CppiaJitConcatNull/Main.hx b/test/regression/CppiaJitConcatNull/Main.hx new file mode 100644 index 000000000..e1b4cc7a2 --- /dev/null +++ b/test/regression/CppiaJitConcatNull/Main.hx @@ -0,0 +1,13 @@ +// Runs the script interpreted and then under the JIT +function main() { + var script = haxe.io.Path.join([haxe.io.Path.directory(Sys.programPath()), "script.cppia"]); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + Sys.println(jit ? "jit" : "interpreted"); + try { + cpp.cppia.Host.runFile(script); + } catch (e:Dynamic) { + Sys.println("error: " + e); + } + } +} diff --git a/test/regression/CppiaJitConcatNull/Script.hx b/test/regression/CppiaJitConcatNull/Script.hx new file mode 100644 index 000000000..04b7cb4ea --- /dev/null +++ b/test/regression/CppiaJitConcatNull/Script.hx @@ -0,0 +1,30 @@ +// Under the JIT, concat with a null array, which throws Null Object Reference +// with HXCPP_CHECK_POINTER, went past the script's catch and ended the host, +// on a typed array and on Array. Interpreted, the script catches it. +class Script { + static var noInts:Array = null; + static var noStrings:Array = null; + static var noAny:Array = null; + + static function main() { + var ints = [1, 2]; + var strings = ["a"]; + var any:Array = [1, "b"]; + try { + Sys.println("ints: ran, " + ints.concat(noInts)); + } catch (e:Dynamic) { + Sys.println("ints: caught " + e); + } + try { + Sys.println("strings: ran, " + strings.concat(noStrings)); + } catch (e:Dynamic) { + Sys.println("strings: caught " + e); + } + try { + Sys.println("any: ran, " + any.concat(noAny)); + } catch (e:Dynamic) { + Sys.println("any: caught " + e); + } + Sys.println("valid: " + ints.concat([3]) + " " + any.concat([true])); + } +} diff --git a/test/regression/CppiaJitConcatNull/build.hxml b/test/regression/CppiaJitConcatNull/build.hxml new file mode 100644 index 000000000..7a34a9e02 --- /dev/null +++ b/test/regression/CppiaJitConcatNull/build.hxml @@ -0,0 +1,8 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no +-D HXCPP_CHECK_POINTER diff --git a/test/regression/CppiaJitConcatNull/stdout.txt b/test/regression/CppiaJitConcatNull/stdout.txt new file mode 100644 index 000000000..e0ddb8553 --- /dev/null +++ b/test/regression/CppiaJitConcatNull/stdout.txt @@ -0,0 +1,10 @@ +interpreted +ints: caught Null Object Reference +strings: caught Null Object Reference +any: caught Null Object Reference +valid: [1,2,3] [1,b,true] +jit +ints: caught Null Object Reference +strings: caught Null Object Reference +any: caught Null Object Reference +valid: [1,2,3] [1,b,true] From 7fdae8c0c8e28bc0004d8cf2d428083aa6abff38 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 07:29:26 -0500 Subject: [PATCH 25/31] Skip a loading cppia module's unmade types when the GC moves objects A loading module is listed with the others from the start, and its list of types is sized before each type is made. Marking it already skipped the entries not yet made, but visiting it, which updates its pointers after a collection moved objects, read them all, so with HXCPP_GC_MOVING a collection that moved objects while a module was making its types crashed the host. The visit now skips them too. The new test loads a module 20 times, interpreted and under the JIT, in a debug build with HXCPP_GC_DEBUG_ALWAYS_MOVE that collects every three allocations while each module loads. It crashed in each of six runs before this change and passed in each of twelve after it. --- Changes.md | 1 + src/hx/cppia/CppiaModule.cpp | 3 ++- test/regression/CppiaLoadMovingGc/Main.hx | 23 ++++++++++++++++++++ test/regression/CppiaLoadMovingGc/Script.hx | 19 ++++++++++++++++ test/regression/CppiaLoadMovingGc/build.hxml | 11 ++++++++++ test/regression/CppiaLoadMovingGc/stdout.txt | 3 +++ 6 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 test/regression/CppiaLoadMovingGc/Main.hx create mode 100644 test/regression/CppiaLoadMovingGc/Script.hx create mode 100644 test/regression/CppiaLoadMovingGc/build.hxml create mode 100644 test/regression/CppiaLoadMovingGc/stdout.txt diff --git a/Changes.md b/Changes.md index 67372ba31..7e0f9e198 100644 --- a/Changes.md +++ b/Changes.md @@ -26,6 +26,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed cppia negating an Int as a Float wherever the result was not used as an Int, interpreted and under the JIT: -x with x = 0 printed as "-0", -(-2147483648) gave 2147483648 instead of wrapping, and Type.typeof gave TFloat, as Dynamic, in a String or as a Float * Fixed interpreted cppia giving contains and remove on an Array as an Int where they became Dynamic or a String: "" + a.contains(x) was "1" and Std.isOfType(a.remove(x), Bool) false. The fix for the typed arrays had missed Array's builtins * Fixed concat with a null array under the cppia JIT ending the process where it throws Null Object Reference (with HXCPP_CHECK_POINTER, or HXCPP_CATCH_SEGV as the Cppia host has), on typed arrays and Array: the native call had no exception bridge. The script catches it, as it does interpreted +* Fixed loading a cppia module crashing with HXCPP_GC_MOVING when a collection that moved objects came while the module was making its types: updating the loading module's pointers read the types not yet made libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/cppia/CppiaModule.cpp b/src/hx/cppia/CppiaModule.cpp index 38334c1c7..f8be08f8f 100644 --- a/src/hx/cppia/CppiaModule.cpp +++ b/src/hx/cppia/CppiaModule.cpp @@ -308,7 +308,8 @@ void CppiaModule::visit(hx::VisitContext *__inCtx) { HX_VISIT_MEMBER(strings); for(int i=0;ivisit(__inCtx); + if (types[i]) /* Not yet made while the module loads */ + types[i]->visit(__inCtx); for(int i=0;ivisit(__inCtx); for(int i=0;i>(); + static var values = new haxe.ds.IntMap(); + static var list = new haxe.ds.List(); + static var buffer = new StringBuf(); + static var bytes = haxe.io.Bytes.alloc(4); + static var date = Date.now(); + static var reg = ~/a+/; + + public static function main() { + names.set("a", [1]); + values.set(1, "b"); + list.add(1.5); + buffer.add(haxe.Json.stringify({a: 1})); + bytes.set(0, 1); + Sys.println(names.get("a")[0] + values.get(1) + list.first() + buffer.toString() + bytes.get(0) + (date != null) + reg.match("aa")); + } +} diff --git a/test/regression/CppiaLoadMovingGc/build.hxml b/test/regression/CppiaLoadMovingGc/build.hxml new file mode 100644 index 000000000..a66df9fbf --- /dev/null +++ b/test/regression/CppiaLoadMovingGc/build.hxml @@ -0,0 +1,11 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-debug +-D DEBUGSUFFIX= +-D scriptable +--dce no +-D HXCPP_GC_MOVING +-D HXCPP_GC_DEBUG_ALWAYS_MOVE diff --git a/test/regression/CppiaLoadMovingGc/stdout.txt b/test/regression/CppiaLoadMovingGc/stdout.txt new file mode 100644 index 000000000..e103f03b2 --- /dev/null +++ b/test/regression/CppiaLoadMovingGc/stdout.txt @@ -0,0 +1,3 @@ +1b1.5{"a":1}1truetrue +1b1.5{"a":1}1truetrue +loaded 20 times From c5f2a2c66c73f1049aa33e9bac5712f43452d048 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 07:25:58 -0500 Subject: [PATCH 26/31] Keep the heap at its target under churn of objects bigger than a line A collection started because no free block could take an allocation counts the rows of the free blocks left as in use, since the allocator could not use them. It counted from mNextFreeBlockOfSize[0], which only a request of a line or less moves on, so the blocks that bigger requests had taken and filled since were counted too, with all the free rows they had before. Their rows are counted by marking. With a churn of 200 to 4,000-byte objects and almost nothing live, the heap looked full at every other collection and its target doubled, and the groups the collection after gave back were faulted in again. Only the free blocks no allocator owns are counted now. Churning 2,000,000 Bytes of 2,000 bytes reserves 20 MB instead of 41 MB, the same churn of strings after it 20 MB instead of 83 MB (896 MB with HXCPP_GC_MOVING), and 200,000 of those Bytes take 16 ms instead of 91 ms. GcChurnHeapTarget checks the reserved memory stays under 32 MB. --- Changes.md | 1 + src/hx/gc/Immix.cpp | 8 ++++- test/regression/GcChurnHeapTarget/Main.hx | 31 ++++++++++++++++++++ test/regression/GcChurnHeapTarget/build.hxml | 2 ++ test/regression/GcChurnHeapTarget/stdout.txt | 1 + 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 test/regression/GcChurnHeapTarget/Main.hx create mode 100644 test/regression/GcChurnHeapTarget/build.hxml create mode 100644 test/regression/GcChurnHeapTarget/stdout.txt diff --git a/Changes.md b/Changes.md index 7e0f9e198..28fd4aa78 100644 --- a/Changes.md +++ b/Changes.md @@ -27,6 +27,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed interpreted cppia giving contains and remove on an Array as an Int where they became Dynamic or a String: "" + a.contains(x) was "1" and Std.isOfType(a.remove(x), Bool) false. The fix for the typed arrays had missed Array's builtins * Fixed concat with a null array under the cppia JIT ending the process where it throws Null Object Reference (with HXCPP_CHECK_POINTER, or HXCPP_CATCH_SEGV as the Cppia host has), on typed arrays and Array: the native call had no exception bridge. The script catches it, as it does interpreted * Fixed loading a cppia module crashing with HXCPP_GC_MOVING when a collection that moved objects came while the module was making its types: updating the loading module's pointers read the types not yet made +* Fixed the heap growing to twice its working target, and with more churn to hundreds of megabytes, when a program churned objects bigger than a line (128 bytes) with little of them live. A collection that started because no free block could take an allocation counted the free rows of blocks that bigger allocations had already taken and filled as rows in use, so the heap looked full and its target doubled at every other collection, and the groups given back in between were faulted in again. Churning 2,000,000 Bytes of 2,000 bytes with 64 kept reserves 20 MB instead of 41 MB, the same churn of 2,000-character strings after it 20 MB instead of 83 MB (896 MB with HXCPP_GC_MOVING), and churning 200,000 such Bytes takes 16 ms instead of 91 ms (median of 5 processes); a program that churned 1,000 to 2,000-byte Bytes, strings and arrays peaked at 26 MB to 48 MB instead of 187 MB to 896 MB (libhxcpp #256) libhxcpp 6.0.0-rc.2 ------------------------------------------------------------ diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index 6d9159b56..f38134a0d 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -6499,8 +6499,14 @@ class GlobalAllocator size_t freeFraggedRows = 0; if (inFreeIsFragged) { + // The free blocks no allocation could use. Only a request of a line or + // less moves mNextFreeBlockOfSize[0] on, so past it are also blocks + // that bigger requests took and filled, whose rows marking counts: + // counting their free rows from before as well made a heap of dead + // objects look full, and its target doubled at every other collection for(int i=mNextFreeBlockOfSize[0]; iGetFreeRows(); + if (!mFreeBlocks[i]->mOwned) + freeFraggedRows += mFreeBlocks[i]->GetFreeRows(); } // Now all threads have mTopOfStack & mBottomOfStack set. diff --git a/test/regression/GcChurnHeapTarget/Main.hx b/test/regression/GcChurnHeapTarget/Main.hx new file mode 100644 index 000000000..45c892f24 --- /dev/null +++ b/test/regression/GcChurnHeapTarget/Main.hx @@ -0,0 +1,31 @@ +import cpp.vm.Gc; +import haxe.io.Bytes; + +// Churn of objects bigger than a line (128 bytes) with a tiny live set must +// keep the heap at its working target (20 MB by default). A collection started +// because no free block could take an allocation counted the free rows of the +// blocks past the first one smaller requests had not taken yet as in use, and +// only the smallest requests move that point on, so the rows of the blocks the +// churn had taken and filled were counted too: the heap looked full and its +// target doubled at every other collection. Bytes of 2000 bytes reached 41 MB +// reserved, and strings after them 83 MB (openfl/libhxcpp#256). +// +// With HXCPP_GC_GENERATIONAL the heap stays near the nursery's size instead +// (see GcHeapGrows), which is under the bound as well. +function main() { + var most = 0.0; + var ring = [for (i in 0...64) Bytes.alloc(16)]; + for (i in 0...2000000) { + ring[i & 63] = Bytes.alloc(2000); + if (i & 1023 == 0) + most = Math.max(most, Gc.memInfo64(Gc.MEM_INFO_RESERVED)); + } + var base = StringTools.lpad("", "x", 1990); + var strings = [for (i in 0...64) ""]; + for (i in 0...2000000) { + strings[i & 63] = base + i; + if (i & 1023 == 0) + most = Math.max(most, Gc.memInfo64(Gc.MEM_INFO_RESERVED)); + } + Sys.println(most < 32 * 1048576 ? "stayed at the target" : 'grew to ${Std.int(most / 1048576)} MB'); +} diff --git a/test/regression/GcChurnHeapTarget/build.hxml b/test/regression/GcChurnHeapTarget/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/GcChurnHeapTarget/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/GcChurnHeapTarget/stdout.txt b/test/regression/GcChurnHeapTarget/stdout.txt new file mode 100644 index 000000000..cd745b133 --- /dev/null +++ b/test/regression/GcChurnHeapTarget/stdout.txt @@ -0,0 +1 @@ +stayed at the target From 492c249fe96c6c51e7dd4304cf0e0cc22c80f5cc Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 09:30:59 -0500 Subject: [PATCH 27/31] Keep x *= y on a Dynamic from losing y in the cppia JIT For x *= y with x not a Float, the JIT computed y into a float scratch register and then converted x to a Float with a native call, which may change every float scratch register. sljit 0.95 maps the first one to xmm0 on x86-64, where 0.93 used xmm1, and the conversion leaves its result there: on Windows x64, var x:Dynamic = 3; x *= 5 gave 9, and the cppia suite's check for hxcpp #926 failed under the JIT. x is now converted first, as -= and /= already did, and evaluated before y, as Haxe orders it. The new regression test covers *= on Dynamic locals, fields and array elements, with Int, Float and call operands; it gives 9 for the first case without this change. Refs openfl/libhxcpp#419 --- src/hx/cppia/Cppia.cpp | 20 +++++++------ .../CppiaJitDynamicMultAssign/Main.hx | 11 +++++++ .../CppiaJitDynamicMultAssign/Script.hx | 29 +++++++++++++++++++ .../CppiaJitDynamicMultAssign/build.hxml | 7 +++++ .../CppiaJitDynamicMultAssign/stdout.txt | 12 ++++++++ 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 test/regression/CppiaJitDynamicMultAssign/Main.hx create mode 100644 test/regression/CppiaJitDynamicMultAssign/Script.hx create mode 100644 test/regression/CppiaJitDynamicMultAssign/build.hxml create mode 100644 test/regression/CppiaJitDynamicMultAssign/stdout.txt diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index f8b956fef..2acaabb3d 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -3989,18 +3989,20 @@ void genSetter(CppiaCompiler *compiler, const JitVal &ioValue, ExprType exprType inExpr->genCode(compiler, sJitTemp0, etInt); compiler->mult(ioValue, sJitTemp0, ioValue,false); } + else if (exprType==etFloat) + { + inExpr->genCode(compiler, sJitTempF0, etFloat); + compiler->mult(ioValue, sJitTempF0, ioValue,true); + } else { + // Convert the value first, as -= and /= do: the conversion is a + // native call, which may change every float scratch register + JitTemp fval(compiler, jtFloat); + compiler->convert(ioValue, exprType, fval, etFloat); inExpr->genCode(compiler, sJitTempF0, etFloat); - if (exprType==etFloat) - compiler->mult(ioValue, sJitTempF0, ioValue,true); - else - { - JitTemp fval(compiler, jtFloat); - compiler->convert(ioValue, exprType, fval, etFloat); - compiler->mult(sJitTempF0, sJitTempF0, fval, true); - compiler->convert(sJitTempF0, etFloat, ioValue, exprType); - } + compiler->mult(sJitTempF0, sJitTempF0, fval, true); + compiler->convert(sJitTempF0, etFloat, ioValue, exprType); } break; diff --git a/test/regression/CppiaJitDynamicMultAssign/Main.hx b/test/regression/CppiaJitDynamicMultAssign/Main.hx new file mode 100644 index 000000000..24664e8bb --- /dev/null +++ b/test/regression/CppiaJitDynamicMultAssign/Main.hx @@ -0,0 +1,11 @@ +function main() { + var script = haxe.io.Path.join([haxe.io.Path.directory(Sys.programPath()), "script.cppia"]); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + try { + cpp.cppia.Host.runFile(script); + } catch (e:Dynamic) { + Sys.println("error: " + e); + } + } +} diff --git a/test/regression/CppiaJitDynamicMultAssign/Script.hx b/test/regression/CppiaJitDynamicMultAssign/Script.hx new file mode 100644 index 000000000..549c27e12 --- /dev/null +++ b/test/regression/CppiaJitDynamicMultAssign/Script.hx @@ -0,0 +1,29 @@ +class Script { + static function five():Dynamic { + return 5; + } + + static function main() { + // x *= y on a Dynamic: the jit computed y into a float register, then + // converted x with a native call, which can change that register. On + // x86-64 Windows it did, and 3 *= 5 gave 9 + var x:Dynamic = 3; + x *= 5; + Sys.println("int " + x); + var f:Dynamic = 3.5; + f *= 2; + Sys.println("float " + f); + var c:Dynamic = 3; + c *= five(); + Sys.println("call " + c); + var o:{v:Dynamic} = {v: 4}; + o.v *= 2.5; + Sys.println("field " + o.v); + var a:Array = [6]; + a[0] *= 7; + Sys.println("array " + a[0]); + var w:Dynamic = 3; + w = w * 5; + Sys.println("binary " + w); + } +} diff --git a/test/regression/CppiaJitDynamicMultAssign/build.hxml b/test/regression/CppiaJitDynamicMultAssign/build.hxml new file mode 100644 index 000000000..d20d36bd4 --- /dev/null +++ b/test/regression/CppiaJitDynamicMultAssign/build.hxml @@ -0,0 +1,7 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaJitDynamicMultAssign/stdout.txt b/test/regression/CppiaJitDynamicMultAssign/stdout.txt new file mode 100644 index 000000000..d63fe744c --- /dev/null +++ b/test/regression/CppiaJitDynamicMultAssign/stdout.txt @@ -0,0 +1,12 @@ +int 15 +float 7 +call 15 +field 10 +array 42 +binary 15 +int 15 +float 7 +call 15 +field 10 +array 42 +binary 15 From e1f6ac0654fc1167205b3848a88cd1c6481c5c21 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 12:47:06 -0500 Subject: [PATCH 28/31] Keep what a cpp.Pointer field points into alive Haxe 4's haxe.atomic.AtomicInt on C++ is a cpp.Pointer into a one-element array that nothing else holds. The collector ignored cpp.Pointer fields, so an AtomicInt kept in a field read freed memory once the array was collected (HaxeFoundation/hxcpp#1225). Haxe 5 changed AtomicInt instead (HaxeFoundation/haxe#12236), which no Haxe 4 release has. A cpp.Pointer field and a pointer boxed in a Dynamic are now marked as a pointer on the stack is: whatever in the GC heap they point into is kept, and pinned under HXCPP_GC_MOVING. Pointers to memory outside the heap are left alone. HaxeAtomicIntField makes 10,000 atomics and 10,000 boxed pointers in passes of their own, so their lines hold nothing else that lives, and collects while churning one-element arrays. Before, 9,999 atomics and 9,996 pointers read freed memory; now none do, at 32 and 64 bits and with HXCPP_GC_MOVING, GENERATIONAL, NURSERY, BIG_BLOCKS, DEBUG_ALWAYS_MOVE and INCREMENTAL. --- Changes.md | 1 + include/hx/GC.h | 3 + include/hx/GcTypeInference.h | 8 +++ src/Dynamic.cpp | 9 ++- src/hx/gc/Immix.cpp | 12 ++++ test/regression/HaxeAtomicIntField/Main.hx | 61 +++++++++++++++++++ test/regression/HaxeAtomicIntField/build.hxml | 2 + test/regression/HaxeAtomicIntField/stdout.txt | 2 + 8 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 test/regression/HaxeAtomicIntField/Main.hx create mode 100644 test/regression/HaxeAtomicIntField/build.hxml create mode 100644 test/regression/HaxeAtomicIntField/stdout.txt diff --git a/Changes.md b/Changes.md index 28fd4aa78..70a31fab8 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating * Fixed an Emscripten build failing to link (undefined symbol: __wasm_lpad_context) when -D HXCPP_LINK_NO_EXCEPTIONS was added to a project built before without it: the define also compiles without -fwasm-exceptions, but the build tool took every HXCPP_LINK_ define for a link option and compiled nothing again. It recompiles now, and once back without it diff --git a/include/hx/GC.h b/include/hx/GC.h index bcbb00f14..863a08e5b 100644 --- a/include/hx/GC.h +++ b/include/hx/GC.h @@ -202,6 +202,9 @@ bool IsWeakRefValid(const HX_CHAR *inPtr); // that points to real, active objects. void MarkConservative(int *inBottom, int *inTop,hx::MarkContext *__inCtx); +// Keeps what a cpp.Pointer points into, as a pointer on the stack would +void MarkConservativePointer(void *inPtr,hx::MarkContext *__inCtx); + // Create/Remove a root. // All statics are explicitly registered - this saves adding the whole data segment diff --git a/include/hx/GcTypeInference.h b/include/hx/GcTypeInference.h index e0500a57c..2d5ea744d 100644 --- a/include/hx/GcTypeInference.h +++ b/include/hx/GcTypeInference.h @@ -35,6 +35,14 @@ template<> inline void MarkMember(cpp::Variant &outT,hx::MarkConte { outT.mark(__inCtx); } +// A cpp.Pointer field into the GC heap keeps what it points into. Haxe 4's +// haxe.atomic.AtomicInt is one into an array nothing else holds, which was +// freed under it (HaxeFoundation/hxcpp#1225) +template inline void MarkMember(cpp::Pointer &outT,hx::MarkContext *__inCtx) +{ + if (outT.ptr) + hx::MarkConservativePointer((void *)outT.ptr,__inCtx); +} template inline void MarkMember(hx::Native &outT,hx::MarkContext *__inCtx) { if (outT.ptr) diff --git a/src/Dynamic.cpp b/src/Dynamic.cpp index 1b9862a6f..137c882b3 100644 --- a/src/Dynamic.cpp +++ b/src/Dynamic.cpp @@ -212,7 +212,7 @@ class Int64Data : public hx::Object class PointerData : public hx::Object { public: - inline void *operator new( size_t inSize, hx::NewObjectType inAlloc=NewObjAlloc,const char *inName="cpp.Pointer") + inline void *operator new( size_t inSize, hx::NewObjectType inAlloc=NewObjContainer,const char *inName="cpp.Pointer") { return hx::Object::operator new(inSize,inAlloc,inName); } PointerData(void *inValue) : mValue(inValue) {}; @@ -238,6 +238,13 @@ class PointerData : public hx::Object return mValue < r ? -1 : mValue==r ? 0 : 1; } + // A boxed pointer into the GC heap keeps what it points into, as a + // cpp.Pointer field does: a script's haxe.atomic.AtomicInt is one + void __Mark(hx::MarkContext *__inCtx) HXCPP_OVERRIDE + { + if (mValue) + hx::MarkConservativePointer(mValue,__inCtx); + } void *mValue; }; diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index f38134a0d..96dd27863 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -7982,6 +7982,18 @@ void MarkConservative(int *inBottom, int *inTop,hx::MarkContext *__inCtx) #endif } +void MarkConservativePointer(void *inPtr,hx::MarkContext *__inCtx) +{ + void *word[1] = { inPtr }; + // MarkConservative reads every pointer-sized word starting on an int in + // the range, so this covers the one word and nothing past it + #ifdef HXCPP_STACK_UP + MarkConservative(reinterpret_cast(word + 1) - 1, reinterpret_cast(word) + 1, __inCtx); + #else + MarkConservative(reinterpret_cast(word), reinterpret_cast(word + 1), __inCtx); + #endif +} + } // namespace hx diff --git a/test/regression/HaxeAtomicIntField/Main.hx b/test/regression/HaxeAtomicIntField/Main.hx new file mode 100644 index 000000000..39e32058e --- /dev/null +++ b/test/regression/HaxeAtomicIntField/Main.hx @@ -0,0 +1,61 @@ +import haxe.atomic.AtomicInt; + +// An AtomicInt kept in a field must outlive collections. Haxe 4's own +// AtomicInt is a cpp.Pointer into a one-element array nothing else keeps, so +// the collector freed the array and the atomic read whatever reused its +// memory (HaxeFoundation/hxcpp#1225). A pointer boxed in a Dynamic field +// must keep what it points into the same way +class Holder { + public var count:AtomicInt; + public var boxed:Dynamic; + + public function new() {} +} + +class Main { + static inline var COUNT = 10000; + + static var holders:Array; + + static function fill() { + // The atomics and arrays are made in passes of their own, so the lines + // they are in hold nothing else that lives + for (i in 0...COUNT) + holders[i].count = new AtomicInt(i); + var arrays = [for (i in 0...COUNT) [i * 2]]; + for (i in 0...COUNT) { + var pointer:cpp.Pointer = cpp.Pointer.ofArray(arrays[i]); + holders[i].boxed = pointer; + } + } + + static function churn() { + var junk = []; + for (i in 0...200000) + junk.push([0x5a5a5a5a]); + return junk.length; + } + + static function main() { + holders = [for (i in 0...COUNT) new Holder()]; + fill(); + for (round in 0...4) { + cpp.vm.Gc.run(true); + churn(); + } + cpp.vm.Gc.run(true); + + var wrongAtomic = 0; + var wrongBoxed = 0; + for (i in 0...COUNT) { + var h = holders[i]; + if (h.count.add(1) != i || h.count.load() != i + 1) + wrongAtomic++; + var pointer:cpp.Pointer = h.boxed; + if (pointer.value != i * 2) + wrongBoxed++; + } + Sys.println('atomics wrong: $wrongAtomic'); + Sys.println('boxed pointers wrong: $wrongBoxed'); + } +} diff --git a/test/regression/HaxeAtomicIntField/build.hxml b/test/regression/HaxeAtomicIntField/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/HaxeAtomicIntField/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/HaxeAtomicIntField/stdout.txt b/test/regression/HaxeAtomicIntField/stdout.txt new file mode 100644 index 000000000..2984a3244 --- /dev/null +++ b/test/regression/HaxeAtomicIntField/stdout.txt @@ -0,0 +1,2 @@ +atomics wrong: 0 +boxed pointers wrong: 0 From 34c163b13e75b9296d9fffe439c23b6d2cc28f21 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 17:31:53 -0500 Subject: [PATCH 29/31] Share the counter and flag atomically in SafePointInGcFreeZone The test read and wrote plain statics from two threads, which ThreadSanitizer reports as a race and fails the job with exit code 66. --- test/regression/SafePointInGcFreeZone/Main.hx | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/regression/SafePointInGcFreeZone/Main.hx b/test/regression/SafePointInGcFreeZone/Main.hx index 812fd5371..3bb6dbb37 100644 --- a/test/regression/SafePointInGcFreeZone/Main.hx +++ b/test/regression/SafePointInGcFreeZone/Main.hx @@ -1,4 +1,6 @@ import cpp.vm.Gc; +import haxe.atomic.AtomicBool; +import haxe.atomic.AtomicInt; import sys.thread.Condition; import sys.thread.Lock; import sys.thread.Mutex; @@ -9,17 +11,18 @@ import sys.thread.Thread; // stop paused the thread as if it were outside a zone: during a collection the // process ended with "Bad Allocation while collecting", and otherwise the thread // waited for a release the collector never sends to a thread in a zone. A -// thread in a zone is already safe, so the stop now does nothing there. +// thread in a zone is already safe, so the stop now does nothing there. The +// counter and flag the threads share are atomic so ThreadSanitizer passes it class Main { - static var count = 0; - static var stop = false; + static var count = new AtomicInt(0); + static var stop = new AtomicBool(false); static function main() { var finished = new Lock(); var mutex = new Mutex(); var condition = new Condition(); Thread.create(() -> { - while (!stop) { + while (!stop.load()) { Gc.enterGCFreeZone(); mutex.acquire(); mutex.release(); @@ -30,7 +33,7 @@ class Main { if (condition.tryAcquire()) condition.release(); Gc.exitGCFreeZone(); - count++; + count.add(1); } finished.release(); }); @@ -40,17 +43,17 @@ class Main { for (i in 0...2000) junk.push([i]); Gc.run(false); - var before = count; + var before = count.load(); Sys.sleep(0.005); - if (count == before) { + if (count.load() == before) { Sys.sleep(2.0); - if (count == before) { + if (count.load() == before) { stalled = true; break; } } } - stop = true; + stop.store(true); if (stalled) { Sys.println("the thread in the GC-free zone stopped"); Sys.exit(1); From 72a6c40fc31b1c28d6984791578f1d7e5eb7f67c Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Wed, 18 Nov 2026 21:57:57 -0500 Subject: [PATCH 30/31] Cap Socket.select's timeout at 1e8 seconds for macOS macOS' select fails with EINVAL for a timeval past 1e8 seconds, so the 2^31-1 second cap threw "Select error" there for a timeout of 3e9 s or infinity. The cap is 1e8 seconds on every system, over three years. Test: SocketSelectLongTimeout, which failed on macOS arm64 CI, passes on Windows x64 3 of 3. --- Changes.md | 2 +- src/hx/libs/std/Socket.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Changes.md b/Changes.md index 70a31fab8..54466e163 100644 --- a/Changes.md +++ b/Changes.md @@ -10,7 +10,7 @@ libhxcpp 6.0.0 (unreleased) * Fixed an ARM64 Windows build with HXCPP_MSVC_VER naming the Visual Studio to use: on an x64 machine the setup looked for vcvarsx64_arm64.bat, which Visual Studio calls vcvarsamd64_arm64.bat, and stopped, and on an ARM64 machine it found the ARM64 script and then ran the x64 one, compiling for x64. It runs the ARM64 script for the host * Fixed a build whose strip or manifest step failed after the link succeeding when run again, with the output unstripped or without its manifest: the link was recorded as done before those steps ran, so the next build had nothing to do. A failure after the link now forgets it, and the next build links again * Each thread object no longer holds a 64 KB buffer below API level 500: only Haxe 5's hx::thread::Scratch uses it, yet with Haxe 4 every Haxe thread kept one while it ran, and so did every native thread that asked for its sys.thread.Thread, for as long as Haxe's own thread list holds it, which for a native thread is until the process ends -* Fixed Socket.select returning at once with nothing ready for a timeout past 2^31 seconds, Math.POSITIVE_INFINITY included: the timeout overflowed an int of seconds. It is capped at 2^31-1 seconds, as Socket.setTimeout's is, and a NaN timeout waits for nothing. Windows' Poll, which takes the same timeout, is fixed with it +* Fixed Socket.select returning at once with nothing ready for a timeout past 2^31 seconds, Math.POSITIVE_INFINITY included: the timeout overflowed an int of seconds. It is capped at 10^8 seconds (over three years), past which macOS' select fails, and a NaN timeout waits for nothing. Windows' Poll, which takes the same timeout, is fixed with it * Fixed Sys.sleep returning at once, or after a much shorter time, for a duration that did not fit the native call: on Windows from 2^32 milliseconds, about 49.7 days, and past 2^31 seconds elsewhere. Sys.sleep(Math.POSITIVE_INFINITY) returned at once instead of sleeping for good. A long sleep goes in parts that fit * Fixed Lock.wait, Semaphore.acquire, Semaphore.tryAcquire with a timeout and a blocking Deque.pop ending the process with "GCFree Zone mismatch" when called inside a GC-free zone the caller had entered, as with cpp.vm.Gc.enterGCFreeZone: when they blocked they left that zone, so the caller's own exit failed. They keep the caller's zone, as Mutex and Condition already did * Fixed Mutex and Condition ending the process with "Bad Allocation while collecting" when a thread used them inside a GC-free zone it had entered, as with cpp.vm.Gc.enterGCFreeZone, while another thread collected. Their uncontended paths stop at a pending collection, and that stop paused the thread as if it were outside the zone; between collections it could instead wait for ever for a release the collector never sends to a thread in a zone. A thread in a zone is already safe, so cpp.vm.Gc.safePoint and those stops now do nothing there diff --git a/src/hx/libs/std/Socket.cpp b/src/hx/libs/std/Socket.cpp index 17be843d8..11b89ae3e 100644 --- a/src/hx/libs/std/Socket.cpp +++ b/src/hx/libs/std/Socket.cpp @@ -1303,10 +1303,11 @@ static struct timeval *init_timeval( double f, struct timeval *t ) { if (f<0) return 0; // The int conversions below overflowed past 2^31 seconds, and select - // returned at once with nothing ready: the wait is capped, as - // set_timeout caps its own. NaN waits for nothing - if (!(f < 0x7fffffff)) - f = f == f ? 0x7fffffff : 0; + // returned at once with nothing ready: the wait is capped. macOS' + // select fails with EINVAL past 1e8 seconds, so that is the cap + // everywhere (over three years). NaN waits for nothing + if (!(f < 1e8)) + f = f == f ? 1e8 : 0; t->tv_usec = (int)( (f - (int)f ) * 1000000 ); t->tv_sec = (int)f; return t; From 155c9a401a0d67ac6f818b29f0c1184969f6eff9 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 00:35:35 -0500 Subject: [PATCH 31/31] Read a lazily compiled cppia function's code with acquire A thread calling a function the lazy JIT was compiling on another read compiled with a plain load, while the compiler set it under its lock after a release fence. ThreadSanitizer, which does not model the fence, reported the race in CppiaLazyJit. compiled is now read with an acquire load and set with a release store, through getCompiled and setCompiled, and each caller calls the code it read rather than reading it again. Test: the Cppia regressions 58 of 58, the cppia suite under the interpreter and the JIT, and CppiaLazyJit 8 of 8 on Windows x64. --- src/hx/cppia/Cppia.cpp | 6 +++--- src/hx/cppia/Cppia.h | 22 +++++++++++++++++++++ src/hx/cppia/CppiaFunction.cpp | 35 ++++++++++++++++++++++------------ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index 2acaabb3d..3094c88b0 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -740,7 +740,7 @@ static void SLJIT_CALL callLazy(CppiaCtx *ctx, ScriptCallable *function) function->compileLazy(); CATCH_NATIVE if (!ctx->exception) - function->compiled(ctx); + function->getCompiled()(ctx); } // Calls the compiled code in inCode, or when that is null, compiles the function in @@ -770,9 +770,9 @@ void genFunctionCall(ScriptCallable *function, CppiaCompiler *compiler, compiler->add( sJitCtxFrame, sJitFrame, JitVal(framePos) ); // Compiled yet - if (function->compiled) + if (CppiaFunc code = function->getCompiled()) { - compiler->call( JitVal( (void *)(function->compiled)), sJitCtx ); + compiler->call( JitVal( (void *)code), sJitCtx ); } else if (function->lazy) { diff --git a/src/hx/cppia/Cppia.h b/src/hx/cppia/Cppia.h index c38c5d46e..4eb9e2483 100644 --- a/src/hx/cppia/Cppia.h +++ b/src/hx/cppia/Cppia.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -347,6 +348,27 @@ struct ScriptCallable : public CppiaDynamicExpr #ifdef CPPIA_JIT + // A lazy function's compiled is set under the compile lock while other threads + // may be calling it, so it is read with acquire and set with release + inline CppiaFunc getCompiled() const + { + #if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(&compiled, __ATOMIC_ACQUIRE); + #else + CppiaFunc result = *(CppiaFunc const volatile *)&compiled; + std::atomic_thread_fence(std::memory_order_acquire); + return result; + #endif + } + inline void setCompiled(CppiaFunc inCode) + { + #if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(&compiled, inCode, __ATOMIC_RELEASE); + #else + std::atomic_thread_fence(std::memory_order_release); + *(CppiaFunc volatile *)&compiled = inCode; + #endif + } void compile(); void compileLazy(); void genDefaults(CppiaCompiler *compiler); diff --git a/src/hx/cppia/CppiaFunction.cpp b/src/hx/cppia/CppiaFunction.cpp index 9e2f8eaeb..714150d0f 100644 --- a/src/hx/cppia/CppiaFunction.cpp +++ b/src/hx/cppia/CppiaFunction.cpp @@ -240,7 +240,7 @@ static unsigned char *scriptableBase(unsigned char *inStack, int stackSize, static bool isJitted(const ScriptCallable *inFunc) { #ifdef CPPIA_JIT - return inFunc->compiled!=0; + return inFunc->getCompiled()!=0; #else return false; #endif @@ -584,14 +584,18 @@ void ScriptCallable::runFunction(CppiaCtx *ctx) { #ifdef CPPIA_JIT - if (lazy && !compiled) + CppiaFunc code = getCompiled(); + if (lazy && !code) + { compileLazy(); - if (compiled) + code = getCompiled(); + } + if (code) { { AutoFrame frame(ctx); //printf("Running compiled code...\n"); - compiled(ctx); + code(ctx); //printf("Done.\n"); } if (ctx->exception) @@ -635,14 +639,18 @@ void ScriptCallable::runFunction(CppiaCtx *ctx) void ScriptCallable::runFunctionClosure(CppiaCtx *ctx) { #ifdef CPPIA_JIT - if (lazy && !compiled) + CppiaFunc code = getCompiled(); + if (lazy && !code) + { compileLazy(); - if (compiled) + code = getCompiled(); + } + if (code) { { AutoFrame frame(ctx); //printf("Running compiled code...\n"); - compiled(ctx); + code(ctx); //printf("Done.\n"); } if (ctx->exception) @@ -810,9 +818,8 @@ void ScriptCallable::compile() CppiaFunc code = compiler->finishGeneration(); // Another thread can find the code through compiled as soon as it is set - std::atomic_thread_fence(std::memory_order_release); - compiled = code; ownsCompiled = true; + setCompiled(code); delete compiler; } @@ -900,13 +907,17 @@ class CppiaClosure : public hx::Object } #ifdef CPPIA_JIT - if (function->lazy && !function->compiled) + CppiaFunc code = function->getCompiled(); + if (function->lazy && !code) + { function->compileLazy(); - if (function->compiled) + code = function->getCompiled(); + } + if (code) { { AutoFrame frame(ctx); - function->compiled(ctx); + code(ctx); } if (ctx->exception) {