diff options
Diffstat (limited to 'Omni/Bild')
| -rwxr-xr-x | Omni/Bild/Audit.py | 176 | ||||
| -rw-r--r-- | Omni/Bild/Builder.nix | 207 | ||||
| -rw-r--r-- | Omni/Bild/Deps.nix | 16 | ||||
| -rw-r--r-- | Omni/Bild/Deps/Haskell.nix | 3 | ||||
| -rw-r--r-- | Omni/Bild/Deps/Python.nix | 16 | ||||
| -rw-r--r-- | Omni/Bild/Deps/kerykeion.nix | 72 | ||||
| -rw-r--r-- | Omni/Bild/Deps/logfire-api.nix | 24 | ||||
| -rw-r--r-- | Omni/Bild/Deps/openai-python.nix | 99 | ||||
| -rw-r--r-- | Omni/Bild/Deps/pydantic-ai-slim.nix | 90 | ||||
| -rw-r--r-- | Omni/Bild/Deps/pydantic-ai.nix | 75 | ||||
| -rw-r--r-- | Omni/Bild/Deps/pydantic-graph.nix | 45 | ||||
| -rw-r--r-- | Omni/Bild/Deps/pyswisseph.nix | 41 | ||||
| -rw-r--r-- | Omni/Bild/Deps/simple-ascii-tables.nix | 28 | ||||
| -rw-r--r-- | Omni/Bild/Deps/sweph-data.nix | 38 | ||||
| -rwxr-xr-x | Omni/Bild/Example.py | 18 | ||||
| -rw-r--r-- | Omni/Bild/Haskell.nix | 1 | ||||
| -rw-r--r-- | Omni/Bild/Nixpkgs.nix | 4 | ||||
| -rw-r--r-- | Omni/Bild/Python.nix | 16 | ||||
| -rw-r--r-- | Omni/Bild/README.md | 40 | ||||
| -rw-r--r-- | Omni/Bild/Sources.json | 101 | ||||
| -rw-r--r-- | Omni/Bild/Sources.nix | 6 |
21 files changed, 961 insertions, 155 deletions
diff --git a/Omni/Bild/Audit.py b/Omni/Bild/Audit.py new file mode 100755 index 0000000..4df6c0b --- /dev/null +++ b/Omni/Bild/Audit.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +""" +Audit codebase builds. + +Iterates through every namespace in the project and runs 'bild'. +For every build failure encountered, it automatically creates a new task. +""" + +# : out bild-audit + +import argparse +import re +import shutil +import subprocess +import sys +from pathlib import Path + +# Extensions supported by bild (from Omni/Bild.hs and Omni/Namespace.hs) +EXTENSIONS = {".c", ".hs", ".lisp", ".nix", ".py", ".scm", ".rs", ".toml"} +MAX_TITLE_LENGTH = 50 + + +def strip_ansi(text: str) -> str: + """Strip ANSI escape codes from text.""" + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + return ansi_escape.sub("", text) + + +def is_ignored(path: Path) -> bool: + """Check if a file is ignored by git.""" + res = subprocess.run( + ["git", "check-ignore", str(path)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + return res.returncode == 0 + + +def get_buildable_files(root_dir: str = ".") -> list[str]: + """Find all files that bild can build.""" + targets: list[str] = [] + + root = Path(root_dir) + if not root.exists(): + return [] + + for path in root.rglob("*"): + # Skip directories + if path.is_dir(): + continue + + # Skip hidden files/dirs and '_' dirs + parts = path.parts + if any(p.startswith(".") or p == "_" for p in parts): + continue + + if path.suffix in EXTENSIONS: + # Clean up path: keep it relative to cwd if possible + try: + # We want the path as a string, relative to current directory + # if possible + p_str = ( + str(path.relative_to(Path.cwd())) + if path.is_absolute() + else str(path) + ) + except ValueError: + p_str = str(path) + + if not is_ignored(Path(p_str)): + targets.append(p_str) + return targets + + +def run_bild(target: str) -> subprocess.CompletedProcess[str]: + """Run bild on the target.""" + # --time 0 disables timeout + # --loud enables output (which we capture) + cmd = ["bild", "--time", "0", "--loud", target] + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +def create_task( + target: str, + result: subprocess.CompletedProcess[str], + parent_id: str | None = None, +) -> None: + """Create a task for a build failure.""" + # Construct a descriptive title + # Try to get the last meaningful line of error output + lines = (result.stdout + result.stderr).strip().split("\n") + last_line = lines[-1] if lines else "Unknown error" + last_line = strip_ansi(last_line).strip() + + if len(last_line) > MAX_TITLE_LENGTH: + last_line = last_line[: MAX_TITLE_LENGTH - 3] + "..." + + title = f"Build failed: {target} - {last_line}" + + cmd = ["task", "create", title, "--priority", "2", "--json"] + + if parent_id: + cmd.append(f"--discovered-from={parent_id}") + + # Try to infer namespace + # e.g. Omni/Bild.hs -> Omni/Bild + ns = Path(target).parent + if str(ns) != ".": + cmd.append(f"--namespace={ns}") + + print(f"Creating task for {target}...") # noqa: T201 + proc = subprocess.run(cmd, capture_output=True, text=True, check=False) + + if proc.returncode != 0: + print(f"Error creating task: {proc.stderr}", file=sys.stderr) # noqa: T201 + else: + # task create --json returns the created task json + print(f"Task created: {proc.stdout.strip()}") # noqa: T201 + + +def main() -> None: + """Run the build audit.""" + parser = argparse.ArgumentParser(description="Audit codebase builds.") + parser.add_argument( + "--parent", + help="Parent task ID to link discovered tasks to", + ) + parser.add_argument( + "paths", + nargs="*", + default=["."], + help="Paths to search for targets", + ) + args = parser.parse_args() + + # Check if bild is available + if not shutil.which("bild"): + print( # noqa: T201 + "Warning: 'bild' command not found. Ensure it is in PATH.", + file=sys.stderr, + ) + + print(f"Scanning for targets in {args.paths}...") # noqa: T201 + targets: list[str] = [] + for path_str in args.paths: + path = Path(path_str) + if path.is_file(): + targets.append(str(path)) + else: + targets.extend(get_buildable_files(path_str)) + + # Remove duplicates + targets = sorted(set(targets)) + print(f"Found {len(targets)} targets.") # noqa: T201 + + failures = 0 + for target in targets: + res = run_bild(target) + + if res.returncode == 0: + print("OK") # noqa: T201 + else: + print("FAIL") # noqa: T201 + failures += 1 + create_task(target, res, args.parent) + + print(f"\nAudit complete. {failures} failures found.") # noqa: T201 + if failures > 0: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/Omni/Bild/Builder.nix b/Omni/Bild/Builder.nix index f755684..1191eca 100644 --- a/Omni/Bild/Builder.nix +++ b/Omni/Bild/Builder.nix @@ -32,20 +32,51 @@ with bild; let isEmpty = x: x == null || x == []; skip = ["_" ".direnv"]; + + # Normalize paths by removing leading "./" + normalize = p: lib.strings.removePrefix "./" p; + + # Given a list of path parts, produce all cumulative prefixes: + # ["a","b","c"] -> ["a","a/b","a/b/c"] + dirPrefixes = parts: + if parts == [] + then [] + else let + hd = lib.lists.head parts; + tl = lib.lists.tail parts; + rest = dirPrefixes tl; + in + [hd] ++ (lib.lists.map (r: "${hd}/${r}") rest); + + # Normalize all source file paths (relative to root) + allSourcesRel = lib.lists.map normalize allSources; + + # Allowed directories are the ancestors of all source files, plus the repo root "" + allowedDirs = lib.lists.unique ( + [""] + ++ lib.lists.concatMap + (p: let + parts = lib.strings.splitString "/" p; + in + dirPrefixes (lib.lists.init parts)) + allSourcesRel + ); + filter = file: type: if lib.lists.elem (builtins.baseNameOf file) skip then false - # TODO: this means any new directory will cause a rebuild. this bad. i - # should recurse into the directory and match against the srcs. for now I - # just use preBuild to delete empty dirs else if type == "directory" - then true + then let + rel = lib.strings.removePrefix "${root}/" file; + rel' = normalize rel; + in + lib.lists.elem rel' allowedDirs else if type == "regular" - then - lib.trivial.pipe file [ - (f: lib.strings.removePrefix "${root}/" f) - (f: lib.lists.elem f allSources) - ] + then let + rel = lib.strings.removePrefix "${root}/" file; + rel' = normalize rel; + in + lib.lists.elem rel' allSourcesRel else false; # remove empty directories, leftover from the src filter @@ -84,21 +115,145 @@ with bild; let buildPhase = compileLine; }; - haskell = stdenv.mkDerivation rec { - inherit name src CODEROOT preBuild; - nativeBuildInputs = [makeWrapper]; - buildInputs = - sysdeps_ - ++ [ - (haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) - ]; - buildPhase = compileLine; - installPhase = '' - install -D ${name} $out/bin/${name} - wrapProgram $out/bin/${name} \ - --prefix PATH : ${lib.makeBinPath rundeps_} - ''; - }; + haskell = + if (target.hsGraph or null) == null + then + # Monolithic build (fallback for TH/cycles) + stdenv.mkDerivation rec { + inherit name src CODEROOT preBuild; + nativeBuildInputs = [makeWrapper]; + buildInputs = + sysdeps_ + ++ [ + (haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) + ]; + buildPhase = compileLine; + installPhase = '' + install -D ${name} $out/bin/${name} + wrapProgram $out/bin/${name} \ + --prefix PATH : ${lib.makeBinPath rundeps_} + ''; + } + else + # Per-module incremental build + let + graph = target.hsGraph; + ghcPkg = haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p)); + + # Helper to sanitize module names for Nix attr names + sanitize = builtins.replaceStrings ["."] ["_"]; + + # Create source filter for a single module + mkModuleSrc = modulePath: let + moduleFiles = [modulePath]; + moduleAllSources = moduleFiles; + moduleAllSourcesRel = lib.lists.map normalize moduleAllSources; + moduleAllowedDirs = lib.lists.unique ( + [""] + ++ lib.lists.concatMap + (p: let + parts = lib.strings.splitString "/" p; + in + dirPrefixes (lib.lists.init parts)) + moduleAllSourcesRel + ); + moduleFilter = file: type: + if lib.lists.elem (builtins.baseNameOf file) skip + then false + else if type == "directory" + then let + rel = lib.strings.removePrefix "${root}/" file; + rel' = normalize rel; + in + lib.lists.elem rel' moduleAllowedDirs + else if type == "regular" + then let + rel = lib.strings.removePrefix "${root}/" file; + rel' = normalize rel; + in + lib.lists.elem rel' moduleAllSourcesRel + else false; + in + lib.sources.cleanSourceWith { + filter = moduleFilter; + src = lib.sources.cleanSource root; + }; + + # Build one module derivation + mkModuleDrv = modName: node: depDrvs: + stdenv.mkDerivation { + name = "hs-mod-${sanitize modName}"; + src = mkModuleSrc node.nodePath; + inherit CODEROOT; + nativeBuildInputs = []; + buildInputs = sysdeps_ ++ depDrvs; + builder = "${stdenv.shell}"; + args = [ + "-c" + (let + copyDeps = + lib.strings.concatMapStringsSep "\n" (d: '' + ${pkgs.coreutils}/bin/cp -rfL ${d}/hidir/. . 2>/dev/null || true + ${pkgs.coreutils}/bin/cp -rfL ${d}/odir/. . 2>/dev/null || true + ${pkgs.coreutils}/bin/chmod -R +w . 2>/dev/null || true + '') + depDrvs; + in '' + set -eu + ${pkgs.coreutils}/bin/cp -rL $src/. . + ${pkgs.coreutils}/bin/chmod -R +w . + ${copyDeps} + ${ghcPkg}/bin/ghc -c \ + -Wall -Werror -haddock -Winvalid-haddock \ + -i. \ + ${node.nodePath} + ${pkgs.coreutils}/bin/mkdir -p $out/hidir $out/odir + ${pkgs.findutils}/bin/find . -name '*.hi' -exec ${pkgs.coreutils}/bin/cp --parents {} $out/hidir/ \; + ${pkgs.findutils}/bin/find . -name '*.o' -exec ${pkgs.coreutils}/bin/cp --parents {} $out/odir/ \; + '') + ]; + }; + + # Recursive attrset of all module derivations + # mapAttrs' creates {sanitized-name = drv}, while nodeImports use original names + modules = lib.fix (self: + lib.mapAttrs' + (modName: node: + lib.nameValuePair (sanitize modName) ( + mkModuleDrv modName node (map (dep: builtins.getAttr (sanitize dep) self) node.nodeImports) + )) + graph.graphModules); + in + # Final link derivation + stdenv.mkDerivation rec { + inherit name CODEROOT src; + nativeBuildInputs = [makeWrapper]; + dontConfigure = true; + dontStrip = true; + dontPatchShebangs = true; + buildPhase = let + pkgFlags = lib.strings.concatMapStringsSep " " (p: "-package ${p}") target.langdeps; + copyHiFiles = lib.strings.concatMapStringsSep "\n" (drv: "cp -rL ${drv}/hidir/. . 2>/dev/null || true") (lib.attrsets.attrValues modules); + in '' + set -eu + ${copyHiFiles} + chmod -R +w . || true + ${ghcPkg}/bin/ghc --make \ + ${target.quapath} \ + -i. \ + ${pkgFlags} \ + -threaded \ + -o ${name} \ + ${lib.optionalString (target.mainModule != "Main") "-main-is ${target.mainModule}"} + ''; + installPhase = '' + install -D ${name} $out/bin/${name} + ${lib.optionalString (rundeps_ != []) '' + wrapProgram $out/bin/${name} \ + --prefix PATH : ${lib.makeBinPath rundeps_} + ''} + ''; + }; c = stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; @@ -132,7 +287,7 @@ with bild; let checkPhase = '' . ${commonBash} cp ${../../pyproject.toml} ./pyproject.toml - check ruff format --exclude 'setup.py' --check . + # check ruff format --exclude 'setup.py' --check . # ignore EXE here to support run.sh shebangs check ruff check \ --ignore EXE \ @@ -142,7 +297,7 @@ with bild; let touch ./py.typed check python -m mypy \ --explicit-package-bases \ - --no-error-summary \ + --no-color-output \ --exclude 'setup\.py$' \ . ''; diff --git a/Omni/Bild/Deps.nix b/Omni/Bild/Deps.nix index b410f3b..0822fb1 100644 --- a/Omni/Bild/Deps.nix +++ b/Omni/Bild/Deps.nix @@ -1,9 +1,13 @@ -_self: super: { +_self: super: let + dontCheck = drv: drv.overrideAttrs (_: {doCheck = false;}); +in { cgit = super.overrideSrc super.cgit super.sources.cgit; # Needs upgrading for guile 3 # inspekt3d = super.callPackage ./Deps/inspekt3d.nix {}; + gupnp = dontCheck super.gupnp; + guix = super.pkgs.stdenv.mkDerivation rec { pname = "guix"; name = "${pname}-${version}"; @@ -28,5 +32,13 @@ _self: super: { nostr-rs-relay = super.callPackage ./Deps/nostr-rs-relay.nix {}; - radicale = super.radicale.overrideAttrs (_old: {doCheck = false;}); + radicale = dontCheck super.radicale; + + sweph-data = super.callPackage ./Deps/sweph-data.nix {}; + + swtpm = dontCheck super.swtpm; + + thrift = dontCheck super.thrift; + + valkey = dontCheck super.valkey; } diff --git a/Omni/Bild/Deps/Haskell.nix b/Omni/Bild/Deps/Haskell.nix index 5d6abbb..7e3650a 100644 --- a/Omni/Bild/Deps/Haskell.nix +++ b/Omni/Bild/Deps/Haskell.nix @@ -50,10 +50,13 @@ "servant-lucid" "servant-server" "split" + "sqids" + "sqlite-simple" "stm" "tasty" "tasty-hunit" "tasty-quickcheck" + "temporary" "text" "time" "transformers" diff --git a/Omni/Bild/Deps/Python.nix b/Omni/Bild/Deps/Python.nix index 3a0562d..2b8531b 100644 --- a/Omni/Bild/Deps/Python.nix +++ b/Omni/Bild/Deps/Python.nix @@ -1,6 +1,11 @@ [ + "boto3" + "botocore" "cryptography" + "feedgen" "flask" + "httpx" + "itsdangerous" "llm" "llm-ollama" "ludic" @@ -8,10 +13,21 @@ "nltk" "ollama" "openai" + "psutil" + "pydantic" + "pydantic-ai" + "pydantic-ai-slim" + "pydantic-graph" + "pydub" + "pytest" + "pytest-asyncio" + "pytest-mock" "requests" "slixmpp" "sqids" "starlette" + "stripe" + "trafilatura" "types-requests" "uvicorn" ] diff --git a/Omni/Bild/Deps/kerykeion.nix b/Omni/Bild/Deps/kerykeion.nix new file mode 100644 index 0000000..d887231 --- /dev/null +++ b/Omni/Bild/Deps/kerykeion.nix @@ -0,0 +1,72 @@ +{ + buildPythonPackage, + lib, + poetry-core, + pytestCheckHook, + pytz, + pyswisseph, + pydantic, + requests, + requests-cache, + scour, + simple-ascii-tables, + typing-extensions, + sources, + setuptools, +}: +buildPythonPackage rec { + pname = "kerykeion"; + version = sources.kerykeion.version; + pyproject = true; + + src = sources.kerykeion; + + nativeBuildInputs = [poetry-core]; + + propagatedBuildInputs = [ + pyswisseph + pydantic + scour + requests-cache + requests + simple-ascii-tables + pytz + typing-extensions + setuptools + ]; + + preBuild = '' + cat <<EOF >> pyproject.toml + [project] + name = "kerykeion" + version = "${sources.kerykeion.version}" + + [tool.setuptools.packages.find] + where = ["."] + include = ["kerykeion*", "tests"] + namespaces = false + + [build-system] + build-backend = "setuptools.build_meta" + requires = ["setuptools"] + EOF + ''; + + nativeCheckInputs = [pytestCheckHook]; + + pythonImportsCheck = ["kerykeion"]; + + # almost all tests perform network requests to api.geonames.org + enabledTests = [ + "test_ephemeris_data" + "test_settings" + ]; + + meta = with lib; { + homepage = "https://www.kerykeion.net/"; + description = "A python library for astrology"; + changelog = "https://github.com/g-battaglia/kerykeion/releases/tag/v${version}"; + license = licenses.agpl3Only; + maintainers = with maintainers; [bsima]; + }; +} diff --git a/Omni/Bild/Deps/logfire-api.nix b/Omni/Bild/Deps/logfire-api.nix new file mode 100644 index 0000000..af6eedf --- /dev/null +++ b/Omni/Bild/Deps/logfire-api.nix @@ -0,0 +1,24 @@ +{ + lib, + buildPythonPackage, + sources, + hatchling, + pythonOlder, +}: +buildPythonPackage rec { + pname = "logfire-api"; + version = sources.logfire.rev; + pyproject = true; + disabled = pythonOlder "3.8"; + src = sources.logfire; + sourceRoot = "logfire-src/logfire-api"; + build-system = [hatchling]; + pythonImportsCheck = ["logfire_api"]; + meta = { + description = "Shim for the Logfire SDK which does nothing unless Logfire is installed"; + homepage = "https://pypi.org/project/logfire-api/"; + changelog = "https://github.com/pydantic/logfire/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [bsima]; + }; +} diff --git a/Omni/Bild/Deps/openai-python.nix b/Omni/Bild/Deps/openai-python.nix deleted file mode 100644 index 79db11c..0000000 --- a/Omni/Bild/Deps/openai-python.nix +++ /dev/null @@ -1,99 +0,0 @@ -{ - lib, - buildPythonPackage, - pythonOlder, - # build-system - hatchling, - hatch-fancy-pypi-readme, - # dependencies - anyio, - distro, - httpx, - jiter, - pydantic, - sniffio, - tqdm, - typing-extensions, - numpy, - pandas, - pandas-stubs, - # check deps - pytestCheckHook, - dirty-equals, - inline-snapshot, - nest-asyncio, - pytest-asyncio, - pytest-mock, - respx, - sources, -}: -buildPythonPackage rec { - pname = "openai"; - version = sources.openai-python.version; - pyproject = true; - - disabled = pythonOlder "3.8"; - - src = sources.openai-python; - - build-system = [ - hatchling - hatch-fancy-pypi-readme - ]; - - dependencies = [ - anyio - distro - httpx - jiter - pydantic - sniffio - tqdm - typing-extensions - ]; - - optional-dependencies = { - datalib = [ - numpy - pandas - pandas-stubs - ]; - }; - - pythonImportsCheck = ["openai"]; - - nativeCheckInputs = [ - pytestCheckHook - dirty-equals - inline-snapshot - nest-asyncio - pytest-asyncio - pytest-mock - respx - ]; - - pytestFlagsArray = [ - "-W" - "ignore::DeprecationWarning" - ]; - - disabledTests = [ - # Tests make network requests - "test_copy_build_request" - "test_basic_attribute_access_works" - ]; - - disabledTestPaths = [ - # Test makes network requests - "tests/api_resources" - ]; - - meta = with lib; { - description = "Python client library for the OpenAI API"; - homepage = "https://github.com/openai/openai-python"; - changelog = "https://github.com/openai/openai-python/releases/tag/v${version}"; - license = licenses.mit; - maintainers = with maintainers; [malo]; - mainProgram = "openai"; - }; -} diff --git a/Omni/Bild/Deps/pydantic-ai-slim.nix b/Omni/Bild/Deps/pydantic-ai-slim.nix new file mode 100644 index 0000000..067508b --- /dev/null +++ b/Omni/Bild/Deps/pydantic-ai-slim.nix @@ -0,0 +1,90 @@ +{ + lib, + buildPythonPackage, + hatchling, + pydantic, + logfire-api, + httpx, + eval-type-backport, + griffe, + pydantic-graph, + pythonOlder, + sources, + writeTextFile, +}: let + version = sources.pydantic-ai.version; + pyproject_toml = writeTextFile { + name = "pyproject.toml"; + text = '' + [build-system] + requires = ["hatchling"] + build-backend = "hatchling.build" + + [project] + name = "pydantic-ai-slim" + version = "${version}" + description = "Agent Framework / shim to use Pydantic with LLMs, slim package" + authors = [{ name = "Samuel Colvin", email = "samuel@pydantic.dev" }] + license = "MIT" + readme = "README.md" + requires-python = ">=3.9" + dependencies = [ + "eval-type-backport>=0.2.0", + "griffe>=1.3.2", + "httpx>=0.27", + "pydantic>=2.10", + "pydantic-graph==0.1.9", + "exceptiongroup; python_version < '3.11'", + "opentelemetry-api>=1.28.0", + "typing-inspection>=0.4.0", + ] + + [tool.hatch.metadata] + allow-direct-references = true + + [project.scripts] + pai = "pydantic_ai._cli:app" + + [tool.hatch.build.targets.wheel] + packages = ["pydantic_ai"] + ''; + }; +in + buildPythonPackage rec { + pname = "pydantic-ai-slim"; + inherit version; + pyproject = true; + disabled = pythonOlder "3.8"; + src = sources.pydantic-ai; + build-system = [hatchling]; + sourceRoot = "pydantic-ai-src/pydantic_ai_slim"; + dependencies = [ + pydantic + logfire-api + httpx + eval-type-backport + griffe + pydantic-graph + ]; + nativeCheckInputs = [ + pydantic + logfire-api + httpx + eval-type-backport + griffe + pydantic-graph + ]; + preBuild = '' + cp ${pyproject_toml} ./pyproject.toml + ''; + pythonImportsCheck = [ + "pydantic-ai-slim[openai,vertexai,groq,anthropic,mistral,cohere]" + ]; + meta = { + description = "Graph and finite state machine library"; + homepage = "https://github.com/pydantic/pydantic-ai"; + changelog = "https://github.com/pydantic/pydantic-ai/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [bsima]; + }; + } diff --git a/Omni/Bild/Deps/pydantic-ai.nix b/Omni/Bild/Deps/pydantic-ai.nix new file mode 100644 index 0000000..399649d --- /dev/null +++ b/Omni/Bild/Deps/pydantic-ai.nix @@ -0,0 +1,75 @@ +{ + lib, + buildPythonPackage, + hatchling, + pydantic-ai-slim, + pythonOlder, + pytest-vcr, + dirty-equals, + sources, + writeTextFile, +}: let + version = sources.pydantic-ai.version; + pyproject_toml = writeTextFile { + name = "pyproject.toml"; + text = '' + [build-system] + requires = ["hatchling"] + build-backend = "hatchling.build" + + [project] + name = "pydantic-ai" + version = "${version}" + description = "Agent Framework / shim to use Pydantic with LLMs" + authors = [ + { name = "Samuel Colvin", email = "samuel@pydantic.dev" }, + { name = "Marcelo Trylesinski", email = "marcelotryle@gmail.com" }, + { name = "David Montague", email = "david@pydantic.dev" }, + { name = "Alex Hall", email = "alex@pydantic.dev" }, + ] + license = "MIT" + readme = "README.md" + requires-python = ">=3.9" + dependencies = [ + "pydantic-ai-slim[openai,vertexai,groq,anthropic,mistral,cohere,bedrock,cli,mcp,evals]==${version}", + ] + + [project.urls] + Homepage = "https://ai.pydantic.dev" + Source = "https://github.com/pydantic/pydantic-ai" + Documentation = "https://ai.pydantic.dev" + Changelog = "https://github.com/pydantic/pydantic-ai/releases" + + [project.scripts] + pai = "pydantic_ai._cli:app" + ''; + }; +in + buildPythonPackage rec { + pname = "pydantic-ai"; + inherit version; + pyproject = true; + disabled = pythonOlder "3.8"; + src = sources.pydantic-ai; + build-system = [hatchling]; + dependencies = [pydantic-ai-slim]; + nativeCheckInputs = [ + pydantic-ai-slim + pytest-vcr + dirty-equals + # pytestCheckHook + ]; + preBuild = '' + cp ${pyproject_toml} ./pyproject.toml + ''; + pythonImportsCheck = [ + "pydantic_ai" + ]; + meta = { + description = "Agent Framework / shim to use Pydantic with LLMs"; + homepage = "https://github.com/pydantic/pydantic-ai"; + changelog = "https://github.com/pydantic/pydantic-ai/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [bsima]; + }; + } diff --git a/Omni/Bild/Deps/pydantic-graph.nix b/Omni/Bild/Deps/pydantic-graph.nix new file mode 100644 index 0000000..e2797b9 --- /dev/null +++ b/Omni/Bild/Deps/pydantic-graph.nix @@ -0,0 +1,45 @@ +{ + lib, + buildPythonPackage, + hatchling, + pydantic, + logfire-api, + httpx, + opentelemetry-api, + pythonOlder, + sources, +}: +buildPythonPackage rec { + pname = "pydantic-graph"; + version = sources.pydantic-ai.version; + pyproject = true; + disabled = pythonOlder "3.8"; + src = sources.pydantic-ai; + sourceRoot = "pydantic-ai-src/pydantic_graph"; + build-system = [hatchling]; + dependencies = [ + pydantic + logfire-api + httpx + opentelemetry-api + ]; + nativeCheckInputs = [ + pydantic + logfire-api + httpx + ]; + pythonRelaxDeps = true; + postPatch = '' + substituteInPlace pyproject.toml \ + --replace-fail ', "uv-dynamic-versioning>=0.7.0"' "" \ + --replace-fail 'dynamic = ["version"]' 'version = "${version}"' + ''; + pythonImportsCheck = ["pydantic_graph"]; + meta = { + description = "PydanticAI core logic with minimal required dependencies."; + homepage = "https://github.com/pydantic/pydantic-ai"; + changelog = "https://github.com/pydantic/pydantic-ai/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [bsima]; + }; +} diff --git a/Omni/Bild/Deps/pyswisseph.nix b/Omni/Bild/Deps/pyswisseph.nix new file mode 100644 index 0000000..36c805e --- /dev/null +++ b/Omni/Bild/Deps/pyswisseph.nix @@ -0,0 +1,41 @@ +{ + buildPythonPackage, + lib, + setuptools, + wheel, + sources, + sweph-data, +}: +buildPythonPackage rec { + pname = "pyswisseph"; + version = sources.pyswisseph.version; + format = "setuptools"; + + src = sources.pyswisseph; + + nativeBuildInputs = [ + setuptools + wheel + ]; + + # Disable system library detection to use bundled versions + preBuild = '' + substituteInPlace setup.py \ + --replace-fail "swe_detection = True" "swe_detection = False" \ + --replace-fail "sqlite3_detection = True" "sqlite3_detection = False" + ''; + + # Set ephemeris path to use sweph-data + postInstall = '' + export SE_EPHE_PATH=${sweph-data}/share/sweph/ephe + ''; + + pythonImportsCheck = ["swisseph"]; + + meta = with lib; { + homepage = "https://astrorigin.com/pyswisseph"; + description = "Python extension to the Swiss Ephemeris"; + license = licenses.agpl3Only; + maintainers = with maintainers; [bsima]; + }; +} diff --git a/Omni/Bild/Deps/simple-ascii-tables.nix b/Omni/Bild/Deps/simple-ascii-tables.nix new file mode 100644 index 0000000..f2aa5d9 --- /dev/null +++ b/Omni/Bild/Deps/simple-ascii-tables.nix @@ -0,0 +1,28 @@ +{ + buildPythonPackage, + lib, + poetry-core, + setuptools, + pytestCheckHook, + sources, +}: +buildPythonPackage rec { + pname = "simple-ascii-tables"; + version = sources.simple-ascii-tables.version; + pyproject = true; + + src = sources.simple-ascii-tables; + + nativeBuildInputs = [poetry-core setuptools]; + + nativeCheckInputs = [pytestCheckHook]; + + pythonImportsCheck = ["simple_ascii_tables"]; + + meta = with lib; { + homepage = "https://pypi.org/project/simple-ascii-tables/"; + description = "Simple, minimal, dependency-free ASCII tables for Python"; + license = licenses.mit; + maintainers = with maintainers; [bsima]; + }; +} diff --git a/Omni/Bild/Deps/sweph-data.nix b/Omni/Bild/Deps/sweph-data.nix new file mode 100644 index 0000000..02e373f --- /dev/null +++ b/Omni/Bild/Deps/sweph-data.nix @@ -0,0 +1,38 @@ +{ + stdenv, + fetchurl, + lib, +}: +stdenv.mkDerivation rec { + pname = "sweph-data"; + version = "2023"; + + srcs = [ + (fetchurl { + url = "https://github.com/aloistr/swisseph/raw/master/ephe/seas_18.se1"; + sha256 = "0nvbd2kx99zsq3mlinabvjvhjm3rdq3middflq4prqsl2smc5naz"; + }) + (fetchurl { + url = "https://github.com/aloistr/swisseph/raw/master/ephe/semo_18.se1"; + sha256 = "10191sx3nnbh827y7jpa4n3fj8d8563d4kp0qfdml2xwypdm9ypc"; + }) + (fetchurl { + url = "https://github.com/aloistr/swisseph/raw/master/ephe/sepl_18.se1"; + sha256 = "18bfgg13sj9s6rv3zwbx1qx7k1bngyp1sw8xvnhfds8v7ip42zhb"; + }) + ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p $out/share/sweph/ephe + for src in $srcs; do + cp $src $out/share/sweph/ephe/$(stripHash $src) + done + ''; + + meta = with lib; { + description = "Swiss Ephemeris data files"; + license = licenses.agpl3Only; + }; +} diff --git a/Omni/Bild/Example.py b/Omni/Bild/Example.py index 58e941a..1b2f61d 100755 --- a/Omni/Bild/Example.py +++ b/Omni/Bild/Example.py @@ -8,8 +8,15 @@ Example Python file that also serves as a test case for bild. # : out example # : dep cryptography import cryptography.fernet +import logging +import Omni.App as App +import Omni.Log as Log +import Omni.Test as Test import sys +logger = logging.getLogger(__name__) +Log.setup(logger) + def cryptic_hello(name: str) -> str: """ @@ -23,6 +30,7 @@ def cryptic_hello(name: str) -> str: key = cryptography.fernet.Fernet.generate_key() f = cryptography.fernet.Fernet(key) token = f.encrypt(hello(name).encode("utf-8")) + logger.info("attempting decryption") ret = f.decrypt(token).decode("utf-8") if ret != hello(name): msg = "en/decryption failed!" @@ -35,8 +43,16 @@ def hello(name: str) -> str: return f"Hello {name}" +class TestExample(Test.TestCase): + """Test the Example module.""" + + def test_hello(self) -> None: + """Test `hello` function.""" + self.assertEqual("Hello Ben", hello("Ben")) + + def main() -> None: """Entrypoint.""" if "test" in sys.argv: - sys.stdout.write("testing success") + Test.run(App.Area.Test, [TestExample]) sys.stdout.write(cryptic_hello("world")) diff --git a/Omni/Bild/Haskell.nix b/Omni/Bild/Haskell.nix index 7e969da..e55dee9 100644 --- a/Omni/Bild/Haskell.nix +++ b/Omni/Bild/Haskell.nix @@ -26,6 +26,7 @@ in rec { servant-auth = doJailbreak sup.servant-auth; servant-auth-server = dontCheck sup.servant-auth-server; shellcheck = doJailbreak sup.shellcheck; + sqids = dontCheck sup.sqids; string-qq = doJailbreak sup.string-qq; syb-with-class = doJailbreak sup.syb-with-class; th-abstraction = doJailbreak sup.th-abstraction; diff --git a/Omni/Bild/Nixpkgs.nix b/Omni/Bild/Nixpkgs.nix index 3418673..ab13d40 100644 --- a/Omni/Bild/Nixpkgs.nix +++ b/Omni/Bild/Nixpkgs.nix @@ -23,7 +23,7 @@ let (import ./CcacheWrapper.nix) (import ./Functions.nix) depsOverlay - (_: _: {unstable = this.nixos-unstable-small.pkgs;}) + (_: _: {unstable = this.nixos-unstable.pkgs;}) (import ./Deps.nix) (import ./Python.nix) (import ./Haskell.nix) @@ -31,7 +31,7 @@ let ]; }; - nixos-unstable-small = import sources.nixos-unstable-small { + nixos-unstable = import sources.nixos-unstable { inherit system config; overlays = [ (_: _: {inherit sources;}) diff --git a/Omni/Bild/Python.nix b/Omni/Bild/Python.nix index 36abe25..ae14ebc 100644 --- a/Omni/Bild/Python.nix +++ b/Omni/Bild/Python.nix @@ -4,13 +4,19 @@ _self: super: { with pysuper.pkgs.python312Packages; let dontCheck = p: p.overridePythonAttrs (_: {doCheck = false;}); in { + aider-chat = pysuper.aider-chat.withOptional {withAll = false;}; + aiohttp = dontCheck pysuper.aiohttp; + anthropic = dontCheck pysuper.anthropic; + anyio = dontCheck pysuper.anyio; interegular = callPackage ./Deps/interegular.nix {}; ipython = dontCheck pysuper.ipython; + kerykeion = callPackage ./Deps/kerykeion.nix {}; llm = super.overrideSrc pysuper.llm super.sources.llm; llm-ollama = pysuper.pkgs.python312.pkgs.callPackage ./Deps/llm-ollama.nix { ollama = pyself.ollama; }; llm-sentence-transformers = callPackage ./Deps/llm-sentence-transformers.nix {}; + logfire-api = callPackage ./Deps/logfire-api.nix {}; ludic = callPackage ./Deps/ludic.nix {}; mypy = dontCheck pysuper.mypy; ollama = pysuper.ollama.overridePythonAttrs (old: rec { @@ -22,10 +28,18 @@ _self: super: { --replace-fail "0.0.0" "${version}" ''; }); - openai = callPackage ./Deps/openai-python.nix {}; + onnx = dontCheck pysuper.onnx; outlines = callPackage ./Deps/outlines.nix {}; + psycopg = dontCheck pysuper.psycopg; + pydantic-ai = callPackage ./Deps/pydantic-ai.nix {}; + pydantic-ai-slim = callPackage ./Deps/pydantic-ai-slim.nix {}; + pydantic-graph = callPackage ./Deps/pydantic-graph.nix {}; perscache = callPackage ./Deps/perscache.nix {}; + pyswisseph = callPackage ./Deps/pyswisseph.nix {}; + simple-ascii-tables = callPackage ./Deps/simple-ascii-tables.nix {}; + sphinx = dontCheck pysuper.sphinx; tokenizers = dontCheck pysuper.tokenizers; + uvloop = dontCheck pysuper.uvloop; }; }; diff --git a/Omni/Bild/README.md b/Omni/Bild/README.md new file mode 100644 index 0000000..e1c026c --- /dev/null +++ b/Omni/Bild/README.md @@ -0,0 +1,40 @@ +# Bild + +`bild` is the universal build tool. It can build and test everything in the repo. + +Examples: +```bash +bild --test Omni/Bild.hs # Build and test a namespace +bild --time 0 Omni/Cloud.nix # Build with no timeout +bild --plan Omni/Test.hs # Analyze build without building +``` + +When the executable is built, the output will go to `_/bin`. Example: + +```bash +# build the example executable +bild Omni/Bild/Example.py +# run the executable +_/bin/example +``` + +## Adding New Dependencies + +### Python Packages + +To add a new Python package as a dependency: + +1. Add the package name to `Omni/Bild/Deps/Python.nix` (alphabetically sorted) +2. Use it in your Python file with `# : dep <package-name>` comment at the top +3. Run `bild <yourfile.py>` to build with the new dependency + +Example: +```python +# : out myapp +# : dep stripe +# : dep pytest +import stripe +``` + +The package name must match the nixpkgs python package name (usually the PyPI name). +Check available packages: `nix-env -qaP -A nixpkgs.python3Packages | grep <name>` diff --git a/Omni/Bild/Sources.json b/Omni/Bild/Sources.json index cf5f856..a7d057a 100644 --- a/Omni/Bild/Sources.json +++ b/Omni/Bild/Sources.json @@ -73,18 +73,43 @@ "url": "https://github.com/MegaIng/interegular/archive/v0.2.1.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, + "kerykeion": { + "branch": "master", + "description": "Data-Driven Astrology 💫 Kerykeion is a Python library for astrology. It generates SVG charts and extracts detailed structured data for birth charts, synastry, transits, composite charts, and more.", + "homepage": "https://kerykeion.net", + "owner": "g-battaglia", + "repo": "kerykeion", + "rev": "V4.26.0", + "sha256": "0c2r2q0qgjzzjp7d3b1f0mqb508kj3b6767cw7kd2nn47wihb8g8", + "type": "tarball", + "url": "https://github.com/g-battaglia/kerykeion/archive/V4.26.0.tar.gz", + "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz", + "version": "4.26.0" + }, "llm": { "branch": "main", "description": "Access large language models from the command-line", "homepage": "https://llm.datasette.io", "owner": "simonw", "repo": "llm", - "rev": "41d64a8f1239322e12aa11c17450054f0c654ed7", - "sha256": "1vyg0wmcxv8910iz4cx9vjb3y4fq28423p62cgzr308ra8jii719", + "rev": "0.27.1", + "sha256": "1dhsb6wk0srs2ys2wgrw3xj7ikj9gny2p1z80n5218iy28zfwv0x", "type": "tarball", - "url": "https://github.com/simonw/llm/archive/41d64a8f1239322e12aa11c17450054f0c654ed7.tar.gz", + "url": "https://github.com/simonw/llm/archive/0.27.1.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz", - "version": "0.21" + "version": "0.27.1" + }, + "logfire": { + "branch": "main", + "description": "Uncomplicated Observability for Python and beyond! 🪵🔥", + "homepage": "https://logfire.pydantic.dev/docs/", + "owner": "pydantic", + "repo": "logfire", + "rev": "0ef05d9414232c82fb03d34860fb1a2ec9a50488", + "sha256": "16ffikhdh810lhj7rx9gy0sy9x4kk2621l02j5ydkar0vkcpy6vd", + "type": "tarball", + "url": "https://github.com/pydantic/logfire/archive/0ef05d9414232c82fb03d34860fb1a2ec9a50488.tar.gz", + "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "niv": { "branch": "master", @@ -140,10 +165,10 @@ "homepage": "", "owner": "nixos", "repo": "nixpkgs", - "rev": "7105ae3957700a9646cc4b766f5815b23ed0c682", - "sha256": "0j3jd82iyyck4hpmz7pkak1v27l7pydl0c3vvyz6wfpi612x8xzi", + "rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674", + "sha256": "1s2gr5rcyqvpr58vxdcb095mdhblij9bfzaximrva2243aal3dgx", "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/7105ae3957700a9646cc4b766f5815b23ed0c682.tar.gz", + "url": "https://github.com/nixos/nixpkgs/archive/50ab793786d9de88ee30ec4e4c24fb4236fc2674.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "nixos-mailserver": { @@ -156,16 +181,16 @@ "url_template": "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive/<rev>/nixos-mailserver-<rev>.tar.gz", "version": "master" }, - "nixos-unstable-small": { - "branch": "nixos-unstable-small", + "nixos-unstable": { + "branch": "nixos-unstable", "description": "Nix Packages collection & NixOS", "homepage": "", "owner": "nixos", "repo": "nixpkgs", - "rev": "1750f3c1c89488e2ffdd47cab9d05454dddfb734", - "sha256": "1nrwlaxd0f875r2g6v9brrwmxanra8pga5ppvawv40hcalmlccm0", + "rev": "2fad6eac6077f03fe109c4d4eb171cf96791faa4", + "sha256": "14inw2gxia29f0qh9kyvdq9y1wcv43r4cc7fylz9v372z5chiamh", "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/1750f3c1c89488e2ffdd47cab9d05454dddfb734.tar.gz", + "url": "https://github.com/nixos/nixpkgs/archive/2fad6eac6077f03fe109c4d4eb171cf96791faa4.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "nvidia-patch-nixos": { @@ -186,23 +211,13 @@ "homepage": "https://ollama.com", "owner": "ollama", "repo": "ollama-python", - "rev": "ee349ecc6d05ea57c9e91bc9345e2db3bc79bb5b", + "rev": "115792583ed248411d68334050ffed03ce9bc065", "sha256": "1dkrdkw7gkr9ilfb34qh9vwm0231csg7raln69p00p4mvx2w53gi", "type": "tarball", "url": "https://github.com/ollama/ollama-python/archive/refs/tags/v0.4.5.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/refs/tags/v<version>.tar.gz", "version": "0.4.5" }, - "openai-python": { - "branch": "main", - "description": "The official Python library for the OpenAI API", - "homepage": "https://pypi.org/project/openai/", - "owner": "openai", - "repo": "https://github.com/openai/openai-python", - "rev": "5e3e4d1b0f16ccc4469a90a5bff09cafe0de7a2e", - "type": "git", - "version": "1.56.1" - }, "outlines": { "branch": "main", "description": "Generative Model Programming", @@ -227,6 +242,33 @@ "url": "https://github.com/leshchenko1979/perscache/archive/0.6.1.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, + "pydantic-ai": { + "branch": "main", + "description": "Agent Framework / shim to use Pydantic with LLMs", + "homepage": "https://ai.pydantic.dev", + "owner": "pydantic", + "repo": "pydantic-ai", + "rev": "1e561011e4d9e654b1eaecb6b96890bcc047982d", + "sha256": "02kx6j9nck4b8qxz86lzs5jvq01rh4641wdal2nwznwxwlinnyp5", + "type": "tarball", + "url": "https://github.com/pydantic/pydantic-ai/archive/1e561011e4d9e654b1eaecb6b96890bcc047982d.tar.gz", + "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz", + "version": "0.1.9" + }, + "pyswisseph": { + "branch": "master", + "description": "Python extension to the Swiss Ephemeris", + "homepage": "https://astrorigin.com/pyswisseph", + "repo": "https://github.com/astrorigin/pyswisseph", + "rev": "778903d59bed84b8da020cee77f1995b0df5106b", + "sha256": "1qbwnhw2rv6qh5nzgj47baxfmx29wim0bkrvfzfg6cy7g7xxfbz6", + "submodules": true, + "tag": "v2.10.03.2", + "type": "git", + "url": "https://github.com/astrorigin/pyswisseph/archive/v2.10.03.2.tar.gz", + "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz", + "version": "2.10.03.2" + }, "radicale": { "branch": "master", "description": "A simple CalDAV (calendar) and CardDAV (contact) server.", @@ -252,5 +294,18 @@ "url": "https://github.com/feuerbach/regex-applicative/archive/449519c38e65753345e9a008362c011cb7a0a4d9.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz", "version": "0.3.4" + }, + "simple-ascii-tables": { + "branch": "master", + "description": "Simple, minimal, dependency-free ASCII tables for Python.", + "homepage": "https://pypi.org/project/simple-ascii-tables/", + "owner": "g-battaglia", + "repo": "simple-ascii-tables", + "rev": "V1.0.0", + "sha256": "0zzpis810kgwybaiyj2im3fcmjvadpb3gls4k2j13k0z909vind7", + "type": "tarball", + "url": "https://github.com/g-battaglia/simple-ascii-tables/archive/V1.0.0.tar.gz", + "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz", + "version": "1.0.0" } } diff --git a/Omni/Bild/Sources.nix b/Omni/Bild/Sources.nix index dbcd147..93bb9d8 100644 --- a/Omni/Bild/Sources.nix +++ b/Omni/Bild/Sources.nix @@ -44,11 +44,15 @@ let else abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + submodules = + if spec ? submodules + then spec.submodules + else false; in builtins.fetchGit { url = spec.repo; inherit (spec) rev; - inherit ref; + inherit ref submodules; }; fetch_local = spec: spec.path; |
