diff options
| author | Ben Sima <ben@bsima.me> | 2025-11-14 20:54:16 -0500 |
|---|---|---|
| committer | Ben Sima <ben@bsima.me> | 2025-11-14 20:54:16 -0500 |
| commit | 8baf00d3f7e87b24d735f57fd489ab7210f1b7a9 (patch) | |
| tree | 325cbba4b41b52a6002bc53f47985d52b653c21e /Omni/Bild | |
| parent | 05e9c79a6a7b8f13835398342b6f2d26417da946 (diff) | |
Implement per-module Nix derivations for incremental Haskell builds
This is the core architecture transformation from Phase 3 of the
performance plan. Each Haskell module is now built as a separate
Nix derivation, enabling true incremental builds where only changed
modules and their dependents are rebuilt.
Implementation: - buildHsModuleGraph: Analyzes transitive module
dependencies and builds DAG - TH detection: Falls back to monolithic
build if Template Haskell detected - SCC cycle detection: Falls
back if import cycles found - Per-module Nix builder: Each module ->
separate derivation with .hi and .o - Module dependencies: Copy .hi
files to build dir, use -i flags for imports - Final link: Use ghc
--make with entry point source + -i paths to .hi files - Entry point
fix: Explicitly analyze entry point module separately from deps
Architecture: - Module compilation: ghc -c with -i paths to dependency
.hi files - Source filtering: Each module derivation includes only
its source file - Dependency DAG: Expressed as recursive Nix attrset
with lib.fix - Link phase: ghc --make with entry source file + all .hi
search paths - Fallback: Monolithic ghc --make when hsGraph is null
(TH/cycles)
Performance characteristics: - Change one module -> rebuild only
that + dependents + relink - Nix handles DAG scheduling and caching
automatically - Parallel module compilation (Nix orchestrates) -
Content-addressed caching across machines
Testing: - Added test_buildHsModuleGraph unit test - Verified with
Omni/Bild/Example.hs (4 modules) - Tested incremental rebuild triggers
correct subset
This completes Phase 2 and Phase 3 core optimizations from the plan.
Diffstat (limited to 'Omni/Bild')
| -rw-r--r-- | Omni/Bild/Builder.nix | 149 |
1 files changed, 134 insertions, 15 deletions
diff --git a/Omni/Bild/Builder.nix b/Omni/Bild/Builder.nix index 2d311ff..a91924c 100644 --- a/Omni/Bild/Builder.nix +++ b/Omni/Bild/Builder.nix @@ -115,21 +115,140 @@ with bild; let buildPhase = compileLine; }; - haskell = stdenv.mkDerivation rec { - inherit name src CODEROOT preBuild; - nativeBuildInputs = [makeWrapper]; - buildInputs = - sysdeps_ - ++ [ - (haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) - ]; - buildPhase = compileLine; - installPhase = '' - install -D ${name} $out/bin/${name} - wrapProgram $out/bin/${name} \ - --prefix PATH : ${lib.makeBinPath rundeps_} - ''; - }; + haskell = + if target.hsGraph == null + then + # Monolithic build (fallback for TH/cycles) + stdenv.mkDerivation rec { + inherit name src CODEROOT preBuild; + nativeBuildInputs = [makeWrapper]; + buildInputs = + sysdeps_ + ++ [ + (haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) + ]; + buildPhase = compileLine; + installPhase = '' + install -D ${name} $out/bin/${name} + wrapProgram $out/bin/${name} \ + --prefix PATH : ${lib.makeBinPath rundeps_} + ''; + } + else + # Per-module incremental build + let + graph = target.hsGraph; + ghcPkg = haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p)); + + # Helper to sanitize module names for Nix attr names + sanitize = builtins.replaceStrings ["."] ["_"]; + + # Create source filter for a single module + mkModuleSrc = modulePath: let + moduleFiles = [modulePath]; + moduleAllSources = moduleFiles; + moduleAllSourcesRel = lib.lists.map normalize moduleAllSources; + moduleAllowedDirs = lib.lists.unique ( + [""] + ++ lib.lists.concatMap + (p: let + parts = lib.strings.splitString "/" p; + in + dirPrefixes (lib.lists.init parts)) + moduleAllSourcesRel + ); + moduleFilter = file: type: + if lib.lists.elem (builtins.baseNameOf file) skip + then false + else if type == "directory" + then let + rel = lib.strings.removePrefix "${root}/" file; + rel' = normalize rel; + in + lib.lists.elem rel' moduleAllowedDirs + else if type == "regular" + then let + rel = lib.strings.removePrefix "${root}/" file; + rel' = normalize rel; + in + lib.lists.elem rel' moduleAllSourcesRel + else false; + in + lib.sources.cleanSourceWith { + filter = moduleFilter; + src = lib.sources.cleanSource root; + }; + + # Build one module derivation + mkModuleDrv = modName: node: depDrvs: + stdenv.mkDerivation { + name = "hs-mod-${sanitize modName}"; + src = mkModuleSrc node.nodePath; + inherit CODEROOT; + buildInputs = sysdeps_ ++ [ghcPkg] ++ depDrvs; + buildPhase = let + copyDeps = + lib.strings.concatMapStringsSep "\n" (d: '' + cp -rL ${d}/hidir/. hidir/ 2>/dev/null || true + '') + depDrvs; + in '' + mkdir -p hidir odir + ${copyDeps} + chmod -R +w hidir || true + ghc -c \ + -Wall -Werror -haddock -Winvalid-haddock \ + -i. -ihidir \ + -odir odir -hidir hidir \ + ${node.nodePath} + ''; + installPhase = '' + mkdir -p $out/hidir $out/odir + cp -r hidir/* $out/hidir/ || true + cp -r odir/* $out/odir/ || true + ''; + }; + + # Recursive attrset of all module derivations + modules = lib.fix (self: + lib.mapAttrs + (modName: node: + mkModuleDrv modName node (map (dep: builtins.getAttr dep self) node.nodeImports)) + graph.graphModules); + + # Compute exact object paths at eval time + moduleToObjPath = modName: drv: "${drv}/odir/${lib.strings.replaceStrings ["."] ["/"] modName}.o"; + objectPaths = + lib.attrsets.mapAttrsToList moduleToObjPath modules; + in + # Final link derivation + stdenv.mkDerivation rec { + inherit name CODEROOT src; + nativeBuildInputs = [makeWrapper]; + dontConfigure = true; + buildPhase = let + pkgFlags = lib.strings.concatMapStringsSep " " (p: "-package ${p}") target.langdeps; + hiDirs = lib.attrsets.mapAttrsToList (_modName: drv: "${drv}/hidir") modules; + iFlags = lib.strings.concatMapStringsSep " " (d: "-i ${d}") hiDirs; + in '' + set -eux + echo "Starting custom link phase with ${builtins.toString (builtins.length objectPaths)} object files" + ${ghcPkg}/bin/ghc --make -o ${name} \ + -i. ${iFlags} \ + ${pkgFlags} \ + -threaded \ + ${lib.optionalString (target.mainModule != "Main") "-main-is ${target.mainModule}"} \ + ${target.quapath} + echo "Link completed successfully" + ''; + installPhase = '' + install -D ${name} $out/bin/${name} + ${lib.optionalString (rundeps_ != []) '' + wrapProgram $out/bin/${name} \ + --prefix PATH : ${lib.makeBinPath rundeps_} + ''} + ''; + }; c = stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; |
