diff options
19 files changed, 1067 insertions, 7 deletions
diff --git a/pkgs/development/compilers/julia/default.nix b/pkgs/development/compilers/julia/default.nix new file mode 100644 index 0000000000000..d4cd0ab6b83eb --- /dev/null +++ b/pkgs/development/compilers/julia/default.nix @@ -0,0 +1,20 @@ +{ callPackage }: + +let + juliaWithPackages = callPackage ../../julia-modules {}; + + wrapJulia = julia: julia.overrideAttrs (oldAttrs: { + passthru = (oldAttrs.passthru or {}) // { + withPackages = juliaWithPackages.override { inherit julia; }; + }; + }); + +in + +{ + julia_16-bin = wrapJulia (callPackage ./1.6-bin.nix {}); + julia_18-bin = wrapJulia (callPackage ./1.8-bin.nix {}); + julia_19-bin = wrapJulia (callPackage ./1.9-bin.nix {}); + julia_18 = wrapJulia (callPackage ./1.8.nix {}); + julia_19 = wrapJulia (callPackage ./1.9.nix {}); +} diff --git a/pkgs/development/julia-modules/default.nix b/pkgs/development/julia-modules/default.nix new file mode 100644 index 0000000000000..973a50abe30d6 --- /dev/null +++ b/pkgs/development/julia-modules/default.nix @@ -0,0 +1,193 @@ +{ lib +, callPackage +, runCommand +, fetchFromGitHub +, fetchgit +, fontconfig +, git +, makeWrapper +, writeText +, writeTextFile +, python3 + +# Artifacts dependencies +, fetchurl +, glibc +, pkgs +, stdenv + +, julia + +# Special registry which is equal to JuliaRegistries/General, but every Versions.toml +# entry is augmented with a Nix sha256 hash +, augmentedRegistry ? callPackage ./registry.nix {} + +# Other overridable arguments +, extraLibs ? [] +, precompile ? true +, setDefaultDepot ? true +, makeWrapperArgs ? "" +, packageOverrides ? {} +, makeTransitiveDependenciesImportable ? false # Used to support symbol indexing +}: + +packageNames: + +let + util = callPackage ./util.nix {}; + +in + +let + # Some Julia packages require access to Python. Provide a Nixpkgs version so it + # doesn't try to install its own. + pythonToUse = let + extraPythonPackages = ((callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames); + in (if extraPythonPackages == [] then python3 + else util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages)); + + # Start by wrapping Julia so it has access to Python and any other extra libs. + # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls. + juliaWrapped = runCommand "julia-${julia.version}-wrapped" { buildInputs = [makeWrapper]; inherit makeWrapperArgs; } '' + mkdir -p $out/bin + makeWrapper ${julia}/bin/julia $out/bin/julia \ + --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \ + --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \ + --set PYTHONHOME "${pythonToUse}" \ + --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \ + --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \ + --set JULIA_CONDAPKG_OFFLINE yes \ + --set JULIA_CONDAPKG_BACKEND Null \ + --set JULIA_PYTHONCALL_EXE "@PyCall" + ''; + + # If our closure ends up with certain packages, add others. + packageImplications = { + # Because we want to put PythonCall in PyCall mode so it doesn't try to download + # Python packages + PythonCall = ["PyCall"]; + }; + + # Invoke Julia resolution logic to determine the full dependency closure + packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides; + closureYaml = callPackage ./package-closure.nix { + inherit augmentedRegistry julia packageNames packageImplications; + packageOverrides = packageOverridesRepoified; + }; + + # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call: + # { + # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = { + # src = fetchgit {...}; + # name = "..."; + # version = "..."; + # treehash = "..."; + # }; + # ... + # } + dependencies = runCommand "julia-sources.nix" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml])) git]; } '' + python ${./python}/sources_nix.py \ + "${augmentedRegistry}" \ + '${lib.generators.toJSON {} packageOverridesRepoified}' \ + "${closureYaml}" \ + "$out" + ''; + + # Import the Nix file from the previous step (IFD) and turn each dependency repo into + # a dummy Git repository, as Julia expects. Format the results as a YAML map from + # dependency UUID -> Nix store location: + # { + # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504", + # ... + # } + # This is also the point where we apply the packageOverrides. + dependencyUuidToInfo = import dependencies { inherit fetchgit; }; + fillInOverrideSrc = uuid: info: + if lib.hasAttr info.name packageOverrides then (info // { src = lib.getAttr info.name packageOverrides; }) else info; + dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo); + dependencyUuidToRepoYaml = writeTextFile { + name = "dependency-uuid-to-repo.yml"; + text = lib.generators.toYAML {} dependencyUuidToRepo; + }; + + # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete + # Julia registry containing all the necessary packages + dependencyUuidToInfoYaml = writeTextFile { + name = "dependency-uuid-to-info.yml"; + text = lib.generators.toYAML {} dependencyUuidToInfo; + }; + fillInOverrideSrc' = uuid: info: + if lib.hasAttr info.name packageOverridesRepoified then (info // { src = lib.getAttr info.name packageOverridesRepoified; }) else info; + overridesOnly = lib.mapAttrs fillInOverrideSrc' (lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo); + minimalRegistry = runCommand "minimal-julia-registry" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml])) git]; } '' + python ${./python}/minimal_registry.py \ + "${augmentedRegistry}" \ + "${closureYaml}" \ + '${lib.generators.toJSON {} overridesOnly}' \ + "${dependencyUuidToRepoYaml}" \ + "$out" + ''; + + # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that + # produces the desired Overrides.toml. + artifactsNix = runCommand "julia-artifacts.nix" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml]))]; } '' + python ${./python}/extract_artifacts.py \ + "${dependencyUuidToRepoYaml}" \ + "${closureYaml}" \ + "${juliaWrapped}/bin/julia" \ + "${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \ + '${lib.generators.toJSON {} (import ./extra-libs.nix)}' \ + "$out" + ''; + + # Import the artifacts Nix to build Overrides.toml (IFD) + artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; }; + overridesJson = writeTextFile { + name = "Overrides.json"; + text = lib.generators.toJSON {} artifacts; + }; + overridesToml = runCommand "Overrides.toml" { buildInputs = [(python3.withPackages (ps: with ps; [toml]))]; } '' + python ${./python}/format_overrides.py \ + "${overridesJson}" \ + "$out" + ''; + + # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the + # depot contains package build products (including the precompiled libraries, if precompile=true) + projectAndDepot = callPackage ./depot.nix { + inherit closureYaml extraLibs overridesToml packageImplications precompile; + julia = juliaWrapped; + registry = minimalRegistry; + packageNames = if makeTransitiveDependenciesImportable + then lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo + else packageNames; + }; + +in + +runCommand "julia-${julia.version}-env" { + buildInputs = [makeWrapper]; + + inherit julia; + inherit juliaWrapped; + + # Expose the steps we used along the way in case the user wants to use them, for example to build + # expressions and build them separately to avoid IFD. + inherit dependencies; + inherit closureYaml; + inherit dependencyUuidToInfoYaml; + inherit dependencyUuidToRepoYaml; + inherit minimalRegistry; + inherit artifactsNix; + inherit overridesJson; + inherit overridesToml; + inherit projectAndDepot; +} ('' + mkdir -p $out/bin + makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \ + --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \ + --set-default JULIA_PROJECT "${projectAndDepot}/project" \ + --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib' +'' + lib.optionalString setDefaultDepot '' + sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia +'') diff --git a/pkgs/development/julia-modules/depot.nix b/pkgs/development/julia-modules/depot.nix new file mode 100644 index 0000000000000..18bac9cb46a61 --- /dev/null +++ b/pkgs/development/julia-modules/depot.nix @@ -0,0 +1,85 @@ +{ lib +, runCommand + +, cacert +, curl +, git +, julia +, python3 + +, closureYaml +, extraLibs +, overridesToml +, packageNames +, packageImplications +, precompile +, registry +}: + +runCommand "julia-depot" { + nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs; + inherit precompile registry; + } '' + export HOME=$(pwd) + + echo "Building Julia depot and project with the following inputs" + echo "Julia: ${julia}" + echo "Registry: $registry" + echo "Overrides ${overridesToml}" + + mkdir -p $out/project + export JULIA_PROJECT="$out/project" + + mkdir -p $out/depot/artifacts + export JULIA_DEPOT_PATH="$out/depot" + cp ${overridesToml} $out/depot/artifacts/Overrides.toml + + # These can be useful to debug problems + # export JULIA_DEBUG=Pkg + # export JULIA_DEBUG=loading + + export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt" + + # Only precompile if configured to below + export JULIA_PKG_PRECOMPILE_AUTO=0 + + # Prevent a warning where Julia tries to download package server info + export JULIA_PKG_SERVER="" + + # See if we need to add any extra package names based on the closure + # and the packageImplications. We're using the full closure YAML here since + # it's available, which is slightly weird, but it should work just as well + # for finding the extra packages we need to add + python ${./python}/find_package_implications.py "${closureYaml}" '${lib.generators.toJSON {} packageImplications}' extra_package_names.txt + + # git config --global --add safe.directory '/nix' + export JULIA_PKG_USE_CLI_GIT="true" + + julia -e ' \ + import Pkg + import Pkg.Types: PRESERVE_NONE + + Pkg.Registry.add(Pkg.RegistrySpec(path="${registry}")) + + input = ${lib.generators.toJSON {} packageNames} ::Vector{String} + + if isfile("extra_package_names.txt") + append!(input, readlines("extra_package_names.txt")) + end + + input = unique(input) + + if !isempty(input) + println("Adding packages: " * join(input, " ")) + Pkg.add(input; preserve=PRESERVE_NONE) + Pkg.instantiate() + + if "precompile" in keys(ENV) && ENV["precompile"] != "0" && ENV["precompile"] != "" + Pkg.precompile() + end + end + + # Remove the registry to save space + Pkg.Registry.rm("General") + ' +'' diff --git a/pkgs/development/julia-modules/extra-libs.nix b/pkgs/development/julia-modules/extra-libs.nix new file mode 100644 index 0000000000000..815c67ac7a7dc --- /dev/null +++ b/pkgs/development/julia-modules/extra-libs.nix @@ -0,0 +1,15 @@ +# A map from a Julia package (typically a JLL package) to extra libraries +# that they require from Nix. +# The libraries should be strings evaluated in a "with pkgs" context. + +{ + # Qt5Base_jll + # Needs access to dbus or you get "Cannot find libdbus-1 in your system" + # Repro: build environment with ["Plots"] + # > using Plots; plot(cos, 0, 2pi) + "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" = ["dbus.lib"]; + + # Qt6Base_jll + # Same reason as Qt5Base_jll + "c0090381-4147-56d7-9ebc-da0b1113ec56" = ["dbus.lib"]; +} diff --git a/pkgs/development/julia-modules/extra-python-packages.nix b/pkgs/development/julia-modules/extra-python-packages.nix new file mode 100644 index 0000000000000..30e5179b38200 --- /dev/null +++ b/pkgs/development/julia-modules/extra-python-packages.nix @@ -0,0 +1,22 @@ +{ lib +, python3 +}: + +# This file contains an extra mapping from Julia packages to the Python packages they depend on. + +with lib; + +rec { + packageMapping = { + ExcelFiles = ["xlrd"]; + PyPlot = ["matplotlib"]; + PythonPlot = ["matplotlib"]; + SymPy = ["sympy"]; + }; + + getExtraPythonPackages = names: concatMap (name: let + allCandidates = if hasAttr name packageMapping then getAttr name packageMapping else []; + in + filter (x: hasAttr x python3.pkgs) allCandidates + ) names; +} diff --git a/pkgs/development/julia-modules/extract_artifacts.jl b/pkgs/development/julia-modules/extract_artifacts.jl new file mode 100755 index 0000000000000..dd8b227c661bc --- /dev/null +++ b/pkgs/development/julia-modules/extract_artifacts.jl @@ -0,0 +1,63 @@ + +import Base: UUID +import Pkg.Artifacts: artifact_meta, artifact_names, find_artifacts_toml, load_artifacts_toml, select_downloadable_artifacts +import Pkg.BinaryPlatforms: AbstractPlatform, platform_key_abi, triplet +import Pkg.Operations: gen_build_code +import TOML + +pkg_uuid = UUID(ARGS[1]) +dir = ARGS[2] + +artifacts_toml = find_artifacts_toml(dir) + +if artifacts_toml == nothing + print("") + exit() +end + +platform = platform_key_abi() + +# Using collect_artifacts (from Pkg.jl) is more reliable than calling select_downloadable_artifacts directly. +# collect_artifacts includes support for .pkg/select_artifacts.jl, which may produce different results. +# If we use select_downloadable_artifacts here, then at depot build time it may try to download a different artifact +# and fail. + +# However! The collect_artifacts from Pkg.jl doesn't allow us to pass lazy to select_downloadable_artifacts. +# So we have to paste our own version in here :( + +function collect_artifacts(pkg_root::String; platform::AbstractPlatform) + # Check to see if this package has an (Julia)Artifacts.toml + artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[] + for f in artifact_names + artifacts_toml = joinpath(pkg_root, f) + if isfile(artifacts_toml) + selector_path = joinpath(pkg_root, ".pkg", "select_artifacts.jl") + + # If there is a dynamic artifact selector, run that in an appropriate sandbox to select artifacts + if isfile(selector_path) + # Despite the fact that we inherit the project, since the in-memory manifest + # has not been updated yet, if we try to load any dependencies, it may fail. + # Therefore, this project inheritance is really only for Preferences, not dependencies. + select_cmd = Cmd(`$(gen_build_code(selector_path; inherit_project=true)) --startup-file=no $(triplet(platform))`) + meta_toml = String(read(select_cmd)) + res = TOML.tryparse(meta_toml) + if res isa TOML.ParserError + errstr = sprint(showerror, res; context=stderr) + pkgerror("failed to parse TOML output from running $(repr(selector_path)), got: \n$errstr") + else + push!(artifacts_tomls, (artifacts_toml, TOML.parse(meta_toml))) + end + else + # Otherwise, use the standard selector from `Artifacts` + artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true) + push!(artifacts_tomls, (artifacts_toml, artifacts)) + end + break + end + end + return artifacts_tomls +end + +for (artifacts_toml, artifacts) in collect_artifacts(dir; platform) + TOML.print(artifacts) +end diff --git a/pkgs/development/julia-modules/extract_artifacts_16.jl b/pkgs/development/julia-modules/extract_artifacts_16.jl new file mode 100755 index 0000000000000..af3099278daa0 --- /dev/null +++ b/pkgs/development/julia-modules/extract_artifacts_16.jl @@ -0,0 +1,33 @@ + +import Base: UUID +import Pkg.Artifacts: artifact_meta, find_artifacts_toml, load_artifacts_toml +import Pkg.BinaryPlatforms: platform_key_abi +import TOML + +pkg_uuid = UUID(ARGS[1]) +dir = ARGS[2] + +artifacts_toml = find_artifacts_toml(dir) + +if artifacts_toml == nothing + print("") + exit() +end + +platform = platform_key_abi() + +# Older Julia doesn't provide select_downloadable_artifacts or .pkg/select_artifacts.jl, +# so gather the artifacts the old-fashioned way +artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid=pkg_uuid) + +results = Dict() +for name in keys(artifact_dict) + # Get the metadata about this name for the requested platform + meta = artifact_meta(name, artifact_dict, artifacts_toml; platform=platform) + + # If there are no instances of this name for the desired platform, skip it + meta === nothing && continue + + results[name] = meta +end +TOML.print(results) diff --git a/pkgs/development/julia-modules/package-closure.nix b/pkgs/development/julia-modules/package-closure.nix new file mode 100644 index 0000000000000..2862e30f0b8bb --- /dev/null +++ b/pkgs/development/julia-modules/package-closure.nix @@ -0,0 +1,180 @@ +{ lib +, julia +, python3 +, runCommand + +, augmentedRegistry +, packageNames +, packageOverrides +, packageImplications +}: + +let + # The specific package resolution code depends on the Julia version + # These are pretty similar and could be combined to reduce duplication + resolveCode = if lib.versionOlder julia.version "1.7" then resolveCode1_6 else resolveCode1_8; + + resolveCode1_6 = '' + import Pkg.API: check_package_name + import Pkg.Types: Context!, PRESERVE_NONE, manifest_info, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved + import Pkg.Operations: _resolve, assert_can_add, is_dep, update_package_add + + foreach(pkg -> check_package_name(pkg.name, :add), pkgs) + pkgs = deepcopy(pkgs) # deepcopy for avoid mutating PackageSpec members + Context!(ctx) + + project_deps_resolve!(ctx, pkgs) + registry_resolve!(ctx, pkgs) + stdlib_resolve!(pkgs) + ensure_resolved(ctx, pkgs, registry=true) + + assert_can_add(ctx, pkgs) + + for (i, pkg) in pairs(pkgs) + entry = manifest_info(ctx, pkg.uuid) + pkgs[i] = update_package_add(ctx, pkg, entry, is_dep(ctx, pkg)) + end + + foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs) + + pkgs, deps_map = _resolve(ctx, pkgs, PRESERVE_NONE) +''; + + resolveCode1_8 = '' + import Pkg.API: handle_package_input! + import Pkg.Types: PRESERVE_NONE, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved + import Pkg.Operations: _resolve, assert_can_add, update_package_add + + foreach(handle_package_input!, pkgs) + + # The handle_package_input! call above clears pkg.path, so we have to apply package overrides after + overrides = Dict{String, String}(${builtins.concatStringsSep ", " (lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides)}) + println("Package overrides: ") + println(overrides) + for pkg in pkgs + if pkg.name in keys(overrides) + pkg.path = overrides[pkg.name] + end + end + + project_deps_resolve!(ctx.env, pkgs) + registry_resolve!(ctx.registries, pkgs) + stdlib_resolve!(pkgs) + ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true) + + assert_can_add(ctx, pkgs) + + for (i, pkg) in pairs(pkgs) + entry = Pkg.Types.manifest_info(ctx.env.manifest, pkg.uuid) + is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps]) + pkgs[i] = update_package_add(ctx, pkg, entry, is_dep) + end + + foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs) + + # Save the original pkgs for later. We might need to augment it with the weak dependencies + orig_pkgs = pkgs + + pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_NONE, ctx.julia_version) + + if VERSION >= VersionNumber("1.9") + # Check for weak dependencies, which appear on the RHS of the deps_map but not in pkgs. + # Build up weak_name_to_uuid + uuid_to_name = Dict() + for pkg in pkgs + uuid_to_name[pkg.uuid] = pkg.name + end + weak_name_to_uuid = Dict() + for (uuid, deps) in pairs(deps_map) + for (dep_name, dep_uuid) in pairs(deps) + if !haskey(uuid_to_name, dep_uuid) + weak_name_to_uuid[dep_name] = dep_uuid + end + end + end + + # If we have nontrivial weak dependencies, add each one to the initial pkgs and then re-run _resolve + if !isempty(weak_name_to_uuid) + println("Found weak dependencies: $(keys(weak_name_to_uuid))") + + orig_uuids = Set([pkg.uuid for pkg in orig_pkgs]) + + for (name, uuid) in pairs(weak_name_to_uuid) + if uuid in orig_uuids + continue + end + + pkg = PackageSpec(name, uuid) + + push!(orig_uuids, uuid) + push!(orig_pkgs, pkg) + ctx.env.project.deps[name] = uuid + entry = Pkg.Types.manifest_info(ctx.env.manifest, uuid) + orig_pkgs[length(orig_pkgs)] = update_package_add(ctx, pkg, entry, false) + end + + pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, orig_pkgs, PRESERVE_NONE, ctx.julia_version) + end + end + ''; + + juliaExpression = packageNames: '' + import Pkg + Pkg.Registry.add(Pkg.RegistrySpec(path="${augmentedRegistry}")) + + import Pkg.Types: Context, PackageSpec + + input = ${lib.generators.toJSON {} packageNames} + + if isfile("extra_package_names.txt") + append!(input, readlines("extra_package_names.txt")) + end + + input = unique(input) + + println("Resolving packages: " * join(input, " ")) + + pkgs = [PackageSpec(pkg) for pkg in input] + + ctx = Context() + + ${resolveCode} + + open(ENV["out"], "w") do io + for spec in pkgs + println(io, "- name: " * spec.name) + println(io, " uuid: " * string(spec.uuid)) + println(io, " version: " * string(spec.version)) + if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid) + println(io, " depends_on: ") + for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid]) + println(io, " \"$(dep_name)\": \"$(dep_uuid)\"") + end + end + end + end + ''; +in + +runCommand "julia-package-closure.yml" { buildInputs = [julia (python3.withPackages (ps: with ps; [pyyaml]))]; } '' + mkdir home + export HOME=$(pwd)/home + + echo "Resolving Julia packages with the following inputs" + echo "Julia: ${julia}" + echo "Registry: ${augmentedRegistry}" + + # Prevent a warning where Julia tries to download package server info + export JULIA_PKG_SERVER="" + + julia -e '${juliaExpression packageNames}'; + + # See if we need to add any extra package names based on the closure + # and the packageImplications + python ${./python}/find_package_implications.py "$out" '${lib.generators.toJSON {} packageImplications}' extra_package_names.txt + + if [ -f extra_package_names.txt ]; then + echo "Re-resolving with additional package names" + julia -e '${juliaExpression packageNames}'; + fi +'' diff --git a/pkgs/development/julia-modules/python/dag.py b/pkgs/development/julia-modules/python/dag.py new file mode 100644 index 0000000000000..4cb775cd42cd4 --- /dev/null +++ b/pkgs/development/julia-modules/python/dag.py @@ -0,0 +1,59 @@ + +# This file based on a ChatGPT reponse for the following prompt: +# "can you write code in python to build up a DAG representing +# a dependency tree, and then a function that can return all the +# dependencies of a given node?" + +class Node: + def __init__(self, name): + self.name = name + self.dependencies = set() + + +class DAG: + def __init__(self): + self.nodes = {} + + def add_node(self, node_name, dependencies=None): + if node_name in self.nodes: + raise ValueError(f"Node '{node_name}' already exists in the graph.") + + node = Node(node_name) + if dependencies: + node.dependencies.update(dependencies) + + self.nodes[node_name] = node + + def add_dependency(self, node_name, dependency_name): + if node_name not in self.nodes: + raise ValueError(f"Node '{node_name}' does not exist in the graph.") + + if dependency_name not in self.nodes: + raise ValueError(f"Dependency '{dependency_name}' does not exist in the graph.") + + self.nodes[node_name].dependencies.add(dependency_name) + + def get_dependencies(self, node_name): + if node_name not in self.nodes: + raise ValueError(f"Node '{node_name}' does not exist in the graph.") + + node = self.nodes[node_name] + dependencies = set() + + def traverse_dependencies(current_node): + for dependency in current_node.dependencies: + dependencies.add(dependency) + if dependency in self.nodes: + traverse_dependencies(self.nodes[dependency]) + + traverse_dependencies(node) + return dependencies + + def has_node(self, node_name): + return node_name in self.nodes + + def __str__(self): + graph_str = "" + for node_name, node in self.nodes.items(): + graph_str += f"{node_name} -> {', '.join(node.dependencies)}\n" + return graph_str diff --git a/pkgs/development/julia-modules/python/dedup_overrides.py b/pkgs/development/julia-modules/python/dedup_overrides.py new file mode 100755 index 0000000000000..d1a7337c598dd --- /dev/null +++ b/pkgs/development/julia-modules/python/dedup_overrides.py @@ -0,0 +1,14 @@ + +import json +from pathlib import Path +import sys +import toml + +overrides_path = Path(sys.argv[1]) +out_path = Path(sys.argv[2]) + +with open(overrides_path, "r") as f: + overrides = json.loads(f.read()) + +with open(out_path, "w") as f: + toml.dump(overrides, f) diff --git a/pkgs/development/julia-modules/python/extract_artifacts.py b/pkgs/development/julia-modules/python/extract_artifacts.py new file mode 100755 index 0000000000000..ecbdf10ed7148 --- /dev/null +++ b/pkgs/development/julia-modules/python/extract_artifacts.py @@ -0,0 +1,99 @@ + +import json +from pathlib import Path +import multiprocessing +import subprocess +import sys +import toml +import yaml + +import dag + +dependencies_path = Path(sys.argv[1]) +closure_yaml_path = Path(sys.argv[2]) +julia_path = Path(sys.argv[3]) +extract_artifacts_script = Path(sys.argv[4]) +extra_libs = json.loads(sys.argv[5]) +out_path = Path(sys.argv[6]) + +with open(dependencies_path, "r") as f: + dependencies = yaml.safe_load(f) + dependency_uuids = dependencies.keys() + +with open(closure_yaml_path, "r") as f: + # Build up a map of UUID -> closure information + closure_yaml_list = yaml.safe_load(f) or [] + closure_yaml = {} + for item in closure_yaml_list: + closure_yaml[item["uuid"]] = item + + # Build up a dependency graph of UUIDs + closure_dependencies_dag = dag.DAG() + for uuid, contents in closure_yaml.items(): + if contents.get("depends_on"): + closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values()) + +with open(out_path, "w") as f: + f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") + f.write("rec {\n") + + def process_item(item): + uuid, src = item + lines = [] + artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode()) + if not artifacts: return f' uuid-{uuid} = {{}};\n' + + lines.append(f' uuid-{uuid} = {{') + + for artifact_name, details in artifacts.items(): + if len(details["download"]) == 0: continue + download = details["download"][0] + url = download["url"] + sha256 = download["sha256"] + + git_tree_sha1 = details["git-tree-sha1"] + + depends_on = set() + if closure_dependencies_dag.has_node(uuid): + depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids) + + other_libs = extra_libs.get(uuid, []) + + fixup = f"""fixupPhase = let + libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) + [{" ".join(["uuid-" + x for x in depends_on])}]; + in '' + find $out -type f -executable -exec \ + patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \; + find $out -type f -executable -exec \ + patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \; + ''""" + + derivation = f"""{{ + name = "{artifact_name}"; + src = fetchurl {{ + url = "{url}"; + sha256 = "{sha256}"; + }}; + sourceRoot = "."; + dontConfigure = true; + dontBuild = true; + installPhase = "cp -r . $out"; + {fixup}; + }}""" + + lines.append(f""" "{artifact_name}" = {{ + sha1 = "{git_tree_sha1}"; + path = stdenv.mkDerivation {derivation}; + }};\n""") + + lines.append(' };\n') + + return "\n".join(lines) + + with multiprocessing.Pool(10) as pool: + for s in pool.map(process_item, dependencies.items()): + f.write(s) + + f.write(f""" +}}\n""") diff --git a/pkgs/development/julia-modules/python/find_package_implications.py b/pkgs/development/julia-modules/python/find_package_implications.py new file mode 100644 index 0000000000000..f253d9c1e6268 --- /dev/null +++ b/pkgs/development/julia-modules/python/find_package_implications.py @@ -0,0 +1,24 @@ + +import json +import os +from pathlib import Path +import subprocess +import sys +import yaml + +dependencies_path = Path(sys.argv[1]) +package_implications_json = sys.argv[2] +out_path = Path(sys.argv[3]) + +package_implications = json.loads(package_implications_json) +with open(dependencies_path) as f: + desired_packages = yaml.safe_load(f) or [] + +extra_package_names = [] +for pkg in desired_packages: + if pkg["name"] in package_implications: + extra_package_names.extend(package_implications[pkg["name"]]) + +if len(extra_package_names) > 0: + with open(out_path, "w") as f: + f.write("\n".join(extra_package_names)) diff --git a/pkgs/development/julia-modules/python/format_overrides.py b/pkgs/development/julia-modules/python/format_overrides.py new file mode 100644 index 0000000000000..cc86ba3915287 --- /dev/null +++ b/pkgs/development/julia-modules/python/format_overrides.py @@ -0,0 +1,22 @@ + +import json +from pathlib import Path +import sys +import toml + +overrides_path = Path(sys.argv[1]) +out_path = Path(sys.argv[2]) + +with open(overrides_path, "r") as f: + overrides = json.loads(f.read()) + +result = {} + +for (uuid, artifacts) in overrides.items(): + if len(artifacts) == 0: continue + + for (name, info) in artifacts.items(): + result[info["sha1"]] = info["path"] + +with open(out_path, "w") as f: + toml.dump(result, f) diff --git a/pkgs/development/julia-modules/python/minimal_registry.py b/pkgs/development/julia-modules/python/minimal_registry.py new file mode 100755 index 0000000000000..c9527f0ef809c --- /dev/null +++ b/pkgs/development/julia-modules/python/minimal_registry.py @@ -0,0 +1,98 @@ + +from collections import defaultdict +import copy +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import toml +import util +import yaml + + +registry_path = Path(sys.argv[1]) +desired_packages_path = Path(sys.argv[2]) +package_overrides = json.loads(sys.argv[3]) +dependencies_path = Path(sys.argv[4]) +out_path = Path(sys.argv[5]) + +with open(desired_packages_path, "r") as f: + desired_packages = yaml.safe_load(f) or [] + +uuid_to_versions = defaultdict(list) +for pkg in desired_packages: + uuid_to_versions[pkg["uuid"]].append(pkg["version"]) + +with open(dependencies_path, "r") as f: + uuid_to_store_path = yaml.safe_load(f) + +os.makedirs(out_path) + +registry = toml.load(registry_path / "Registry.toml") +registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions} + +for (uuid, versions) in uuid_to_versions.items(): + if uuid in package_overrides: + info = package_overrides[uuid] + + # Make a registry entry based on the info from the package override + path = Path(info["name"][0].upper()) / Path(info["name"]) + registry["packages"][uuid] = { + "name": info["name"], + "path": str(path), + } + + os.makedirs(out_path / path) + + # Read the Project.yaml from the src + project = toml.load(Path(info["src"]) / "Project.toml") + + # Generate all the registry files + with open(out_path / path / Path("Compat.toml"), "w") as f: + f.write('["%s"]\n' % info["version"]) + # Write nothing in Compat.toml, because we've already resolved everything + with open(out_path / path / Path("Deps.toml"), "w") as f: + f.write('["%s"]\n' % info["version"]) + toml.dump(project["deps"], f) + with open(out_path / path / Path("Versions.toml"), "w") as f: + f.write('["%s"]\n' % info["version"]) + f.write('git-tree-sha1 = "%s"\n' % info["treehash"]) + with open(out_path / path / Path("Package.toml"), "w") as f: + toml.dump({ + "name": info["name"], + "uuid": uuid, + "repo": "file://" + info["src"], + }, f) + + elif uuid in registry["packages"]: + registry_info = registry["packages"][uuid] + name = registry_info["name"] + path = registry_info["path"] + + os.makedirs(out_path / path) + + # Copy some files to the minimal repo unchanged + for f in ["Compat.toml", "Deps.toml"]: + if (registry_path / path / f).exists(): + shutil.copy2(registry_path / path / f, out_path / path) + + # Copy the Versions.toml file, trimming down to the versions we care about + all_versions = toml.load(registry_path / path / "Versions.toml") + versions_to_keep = {k: v for k, v in all_versions.items() if k in versions} + for k, v in versions_to_keep.items(): + del v["nix-sha256"] + with open(out_path / path / "Versions.toml", "w") as f: + toml.dump(versions_to_keep, f) + + # Fill in the local store path for the repo + if not uuid in uuid_to_store_path: continue + package_toml = toml.load(registry_path / path / "Package.toml") + package_toml["repo"] = "file://" + uuid_to_store_path[uuid] + with open(out_path / path / "Package.toml", "w") as f: + toml.dump(package_toml, f) + +with open(out_path / "Registry.toml", "w") as f: + toml.dump(registry, f) diff --git a/pkgs/development/julia-modules/python/sources_nix.py b/pkgs/development/julia-modules/python/sources_nix.py new file mode 100755 index 0000000000000..99a952753e80f --- /dev/null +++ b/pkgs/development/julia-modules/python/sources_nix.py @@ -0,0 +1,67 @@ + +import json +from pathlib import Path +import re +import shutil +import sys +import toml +import util +import yaml + + +registry_path = Path(sys.argv[1]) +package_overrides = json.loads(sys.argv[2]) +desired_packages_path = Path(sys.argv[3]) +out_path = Path(sys.argv[4]) + +with open(desired_packages_path, "r") as f: + desired_packages = yaml.safe_load(f) or [] + +registry = toml.load(registry_path / "Registry.toml") + +def ensure_version_valid(version): + """ + Ensure a version string is a valid Julia-parsable version. + It doesn't really matter what it looks like as it's just used for overrides. + """ + return re.sub('[^0-9\.]','', version) + +with open(out_path, "w") as f: + f.write("{fetchgit}:\n") + f.write("{\n") + for pkg in desired_packages: + uuid = pkg["uuid"] + + if pkg["name"] in package_overrides: + treehash = util.get_commit_info(package_overrides[pkg["name"]])["tree"] + f.write(f""" "{uuid}" = {{ + src = null; # Overridden: will fill in later + name = "{pkg["name"]}"; + version = "{ensure_version_valid(pkg["version"])}"; + treehash = "{treehash}"; + }};\n""") + elif uuid in registry["packages"]: + registry_info = registry["packages"][uuid] + path = registry_info["path"] + packageToml = toml.load(registry_path / path / "Package.toml") + + all_versions = toml.load(registry_path / path / "Versions.toml") + if not pkg["version"] in all_versions: continue + version_to_use = all_versions[pkg["version"]] + + repo = packageToml["repo"] + f.write(f""" "{uuid}" = {{ + src = fetchgit {{ + url = "{repo}"; + rev = "{version_to_use["git-tree-sha1"]}"; + sha256 = "{version_to_use["nix-sha256"]}"; + }}; + name = "{pkg["name"]}"; + version = "{pkg["version"]}"; + treehash = "{version_to_use["git-tree-sha1"]}"; + }};\n""") + else: + # print("Warning: couldn't figure out what to do with pkg in sources_nix.py", pkg) + pass + + f.write("}") diff --git a/pkgs/development/julia-modules/python/util.py b/pkgs/development/julia-modules/python/util.py new file mode 100644 index 0000000000000..34aaab970e594 --- /dev/null +++ b/pkgs/development/julia-modules/python/util.py @@ -0,0 +1,12 @@ + +import os +import subprocess +import tempfile + +def get_commit_info(repo): + with tempfile.TemporaryDirectory() as home_dir: + env_with_home = os.environ.copy() + env_with_home["HOME"] = home_dir + subprocess.check_output(["git", "config", "--global", "--add", "safe.directory", repo], env=env_with_home) + lines = subprocess.check_output(["git", "log", "--pretty=raw"], cwd=repo, env=env_with_home).decode().split("\n") + return dict([x.split() for x in lines if len(x.split()) == 2]) diff --git a/pkgs/development/julia-modules/registry.nix b/pkgs/development/julia-modules/registry.nix new file mode 100644 index 0000000000000..929a805b48c5f --- /dev/null +++ b/pkgs/development/julia-modules/registry.nix @@ -0,0 +1,9 @@ +{ fetchFromGitHub }: + +fetchFromGitHub { + owner = "CodeDownIO"; + repo = "General"; + rev = "baf9e22ecdf97b6424a611ac4a565c6ee60d3f44"; + sha256 = "1nd3x2z8r6578149pbpkx9qw2ajln1kfy7w5kjsnv56v180h0ddf"; + # date = "2023-12-14T12:20:00+00:00"; +} diff --git a/pkgs/development/julia-modules/util.nix b/pkgs/development/julia-modules/util.nix new file mode 100644 index 0000000000000..0c01fcbe82863 --- /dev/null +++ b/pkgs/development/julia-modules/util.nix @@ -0,0 +1,45 @@ +{ git +, runCommand +}: + +{ + # Add packages to a Python environment. Works if you pass something like either + # a) python3 + # b) python3.withPackages (ps: [...]) + # See https://github.com/NixOS/nixpkgs/pull/97467#issuecomment-689315186 + addPackagesToPython = python: packages: + if python ? "env" then python.override (old: { + extraLibs = old.extraLibs ++ packages; + }) + else python.withPackages (ps: packages); + + # Convert an ordinary source checkout into a repo with a single commit + repoifySimple = name: path: + runCommand ''${name}-repoified'' {buildInputs = [git];} '' + mkdir -p $out + cp -r ${path}/. $out + cd $out + chmod -R u+w . + rm -rf .git + git init + git add . -f + git config user.email "julia2nix@localhost" + git config user.name "julia2nix" + git commit -m "Dummy commit" + ''; + + # Convert an dependency source info into a repo with a single commit + repoifyInfo = uuid: info: + runCommand ''julia-${info.name}-${info.version}'' {buildInputs = [git];} '' + mkdir -p $out + cp -r ${info.src}/. $out + cd $out + chmod -R u+w . + rm -rf .git + git init + git add . -f + git config user.email "julia2nix@localhost" + git config user.name "julia2nix" + git commit -m "Dummy commit" + ''; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 8acf7e4753f00..202dd80e5aef3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -16517,17 +16517,17 @@ with pkgs; juniper = callPackage ../development/compilers/juniper { }; + inherit (callPackage ../development/compilers/julia { }) + julia_16-bin + julia_18-bin + julia_19-bin + julia_18 + julia_19; + julia-lts = julia_16-bin; julia-stable = julia_19; julia = julia-stable; - julia_16-bin = callPackage ../development/compilers/julia/1.6-bin.nix { }; - julia_18-bin = callPackage ../development/compilers/julia/1.8-bin.nix { }; - julia_19-bin = callPackage ../development/compilers/julia/1.9-bin.nix { }; - - julia_18 = callPackage ../development/compilers/julia/1.8.nix { }; - julia_19 = callPackage ../development/compilers/julia/1.9.nix { }; - julia-lts-bin = julia_16-bin; julia-stable-bin = julia_19-bin; julia-bin = julia-stable-bin; |