shithub: puzzles

Download patch

ref: 1600818848fc99eb53e0de240670b598ac490fdc
parent: b56c994b721f7cb5c5bcf09c5d695d4ae07d3052
author: Simon Tatham <anakin@pobox.com>
date: Thu Jan 6 04:03:28 EST 2022

Try to fix flakiness in the NestedVM build.

I've had a lot of Puzzles nightly builds fail recently in the NestedVM
stage, with a 'jar' command producing a message along these lines:

java.util.zip.ZipException: attempt to write past end of STORED entry
        at java.base/java.util.zip.ZipOutputStream.write(ZipOutputStream.java:337)
        at jdk.jartool/sun.tools.jar.Main.copy(Main.java:1250)
        at jdk.jartool/sun.tools.jar.Main.copy(Main.java:1263)
        at jdk.jartool/sun.tools.jar.Main.addFile(Main.java:1211)
        at jdk.jartool/sun.tools.jar.Main.create(Main.java:879)
        at jdk.jartool/sun.tools.jar.Main.run(Main.java:319)
        at jdk.jartool/sun.tools.jar.Main.main(Main.java:1681)
        Suppressed: java.util.zip.ZipException: invalid entry size (expected 0 but got 786 bytes)
                at java.base/java.util.zip.ZipOutputStream.closeEntry(ZipOutputStream.java:288)
                at java.base/java.util.zip.ZipOutputStream.finish(ZipOutputStream.java:361)
                at java.base/java.util.zip.DeflaterOutputStream.close(DeflaterOutputStream.java:238)
                at java.base/java.util.zip.ZipOutputStream.close(ZipOutputStream.java:378)
                at jdk.jartool/sun.tools.jar.Main.create(Main.java:854)
                ... 2 more

It's hard to work out exactly what this error dump means, and
web-searching for the error message isn't much help because the same
exception can occur in application code using java.util.zip, and most
mentions on the web are about that, and not about what I want to know,
which is why it might happen in the 'jar' program in particular.

However, the clues visible in that message suggest that 'jar' had
somehow got confused about the size of one of the files it was adding
to the jar archive, in that it initially decided it was 0 bytes long
and later found it was longer. That suggests a problem of excessive
parallelism between the build steps, perhaps due to a missing
dependency in the makefile, which might plausibly cause the 'jar' step
to be running already while some file it needs to read is still being
written. (Which would also explain why it doesn't happen every time.)

An eyeball review of cmake/platforms/nestedvm.cmake didn't find any
obvious missing dependencies. But I vaguely remembered that in some
other context I'd had trouble with cmake 'add_custom_command'. So in
this commit I replace all those custom commands with custom _targets_,
listing the previous OUTPUT files as BYPRODUCTS. And then the
dependencies are written using the target names, instead of the file
names.

I don't fully understand why this should make a difference. But it
seems more reliable in a soak test, and still builds the right things,
so I'll commit it and see if it makes the flakiness in the actual
nightly builds stop happening.

--- a/cmake/platforms/nestedvm.cmake

+++ b/cmake/platforms/nestedvm.cmake

@@ -5,7 +5,8 @@

   "Main-Class: PuzzleApplet\n")

 include(FindJava)

-add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/PuzzleApplet.class

+add_custom_target(nvm-puzzle-applet

+  BYPRODUCTS ${CMAKE_BINARY_DIR}/PuzzleApplet.class

   COMMAND ${Java_JAVAC_EXECUTABLE}

     -source 1.7 -target 1.7 -d . -cp ${NESTEDVM}/build

     ${CMAKE_SOURCE_DIR}/PuzzleApplet.java

@@ -18,9 +19,12 @@

 function(set_platform_gui_target_properties TARGET)

   set(build_subdir ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-tmp)

-  add_custom_command(OUTPUT ${build_subdir}

+  add_custom_target(${TARGET}-nvm-build-subdir

+    BYPRODUCTS ${build_subdir}

     COMMAND ${CMAKE_COMMAND} -E make_directory ${build_subdir})

-  add_custom_command(OUTPUT ${build_subdir}/PuzzleApplet.class

+

+  add_custom_target(${TARGET}-nvm-symlinks

+    BYPRODUCTS ${build_subdir}/PuzzleApplet.class

     COMMAND ${CMAKE_SOURCE_DIR}/cmake/glob-symlinks.py

       ${CMAKE_BINARY_DIR} applet.manifest

       ${CMAKE_BINARY_DIR} PuzzleApplet\\*.class

@@ -31,17 +35,18 @@

       ${NESTEDVM}/build org/ibex/nestedvm/util/Seekable\\*.class

     WORKING_DIRECTORY ${build_subdir}

     DEPENDS

-      ${build_subdir}

-      ${CMAKE_BINARY_DIR}/PuzzleApplet.class

+      ${TARGET}-nvm-build-subdir

+      nvm-puzzle-applet

       ${CMAKE_SOURCE_DIR}/cmake/glob-symlinks.py)

-  add_custom_command(OUTPUT ${build_subdir}/PuzzleEngine.class

+  add_custom_target(${TARGET}-nvm-engine

+    BYPRODUCTS ${build_subdir}/PuzzleEngine.class

     COMMAND ${Java_JAVA_EXECUTABLE}

       -cp ${NESTEDVM}/build:${NESTEDVM}/upstream/build/classgen/build

       org.ibex.nestedvm.Compiler -outformat class -d .

       PuzzleEngine ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}

     DEPENDS

-      ${build_subdir}

+      ${TARGET}-nvm-build-subdir

       ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}

     WORKING_DIRECTORY ${build_subdir})

@@ -51,9 +56,10 @@

       applet.manifest PuzzleEngine.class PuzzleApplet*.class org

       WORKING_DIRECTORY ${build_subdir}

     DEPENDS

-      ${CMAKE_BINARY_DIR}/PuzzleApplet.class

-      ${build_subdir}/PuzzleApplet.class

-      ${build_subdir}/PuzzleEngine.class)

+      nvm-puzzle-applet

+      ${TARGET}-nvm-build-subdir

+      ${TARGET}-nvm-symlinks

+      ${TARGET}-nvm-engine)

 endfunction()

 function(set_platform_puzzle_target_properties NAME TARGET)