Nix shell updates (Nixpkgs 2024-02-23, QMK CLI 1.1.5) (#23143)

shell.nix

@@ -1,29 +1,9 @@
 let
   # We specify sources via Niv: use "niv update nixpkgs" to update nixpkgs, for example.
   sources = import ./util/nix/sources.nix { };
-
-  # `tomlkit` >= 0.8.0 is required to build `jsonschema` >= 4.11.0 (older
-  # version do not support some valid TOML syntax: sdispater/tomlkit#148).  The
-  # updated `tomlkit` must be used by `makeRemoveSpecialDependenciesHook`
-  # inside `poetry2nix`, therefore just providing the updated version through
-  # our `nix/pyproject.toml` does not work, and using an overlay is required.
-  pythonOverlay = final: prev: {
-    python3 = prev.python3.override {
-      packageOverrides = self: super: {
-        tomlkit = super.tomlkit.overridePythonAttrs(old: rec {
-          version = "0.11.4";
-          src = super.fetchPypi {
-            inherit (old) pname;
-            inherit version;
-            sha256 = "sha256-MjWpAQ+uVDI+cnw6wG+3IHUv5mNbNCbjedrsYPvUSoM=";
-          };
-        });
-      };
-    };
-  };
 in
 # However, if you want to override Niv's inputs, this will let you do that.
-{ pkgs ? import sources.nixpkgs { overlays = [ pythonOverlay ]; }
+{ pkgs ? import sources.nixpkgs { }
 , poetry2nix ? pkgs.callPackage (import sources.poetry2nix) { }
 , avr ? true
 , arm ? true
@@ -49,18 +29,22 @@ let
   pythonEnv = poetry2nix.mkPoetryEnv {
     projectDir = ./util/nix;
     overrides = poetry2nix.overrides.withDefaults (self: super: {
-      pillow = super.pillow.overridePythonAttrs(old: {
-        # Use preConfigure from nixpkgs to fix library detection issues and
-        # impurities which can break the build process; this also requires
-        # adding propagatedBuildInputs and buildInputs from the same source.
-        propagatedBuildInputs = (old.buildInputs or []) ++ pkgs.python3.pkgs.pillow.propagatedBuildInputs;
-        buildInputs = (old.buildInputs or []) ++ pkgs.python3.pkgs.pillow.buildInputs;
-        preConfigure = (old.preConfigure or "") + pkgs.python3.pkgs.pillow.preConfigure;
-      });
       qmk = super.qmk.overridePythonAttrs(old: {
         # Allow QMK CLI to run "qmk" as a subprocess (the wrapper changes
         # $PATH and breaks these invocations).
         dontWrapPythonPrograms = true;
+
+        # Fix "qmk setup" to use the Python interpreter from the environment
+        # when invoking "qmk doctor" (sys.executable gets its value from
+        # $NIX_PYTHONEXECUTABLE, which is set by the "qmk" wrapper from the
+        # Python environment, so "qmk doctor" then runs with the proper
+        # $NIX_PYTHONPATH too, because sys.executable actually points to
+        # another wrapper from the same Python environment).
+        postPatch = ''
+          substituteInPlace qmk_cli/subcommands/setup.py \
+            --replace "[Path(sys.argv[0]).as_posix()" \
+              "[Path(sys.executable).as_posix(), Path(sys.argv[0]).as_posix()"
+        '';
       });
     });
   };
@@ -68,7 +52,7 @@ in
 mkShell {
   name = "qmk-firmware";
 
-  buildInputs = [ clang-tools dfu-programmer dfu-util diffutils git pythonEnv niv ]
+  buildInputs = [ clang-tools_11 dfu-programmer dfu-util diffutils git pythonEnv niv ]
     ++ lib.optional avr [
       pkgsCross.avr.buildPackages.binutils
       pkgsCross.avr.buildPackages.gcc8

util/nix/pyproject.toml

@@ -8,7 +8,7 @@ description = ""
 authors = []
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.11"
 appdirs = "*"
 argcomplete = "*"
 colorama = "*"
@@ -34,28 +34,6 @@ pep8-naming = "*"
 pyflakes = "*"
 yapf = "*"
 
-# These dependencies are required by the jsonschema >= 4.11.0 build system, but
-# are not detected automatically; they are also not present in the used Nixpkgs
-# snapshot, so need to be obtained through Poetry.
-hatchling = "*"
-hatch-vcs = "*"
-hatch-fancy-pypi-readme = "*"
-
-# The `pytest` module in the used Nixpkgs snapshot has an upper bound on the
-# `pluggy` dependency, which conflicts with the dependency of the `hatchling`
-# module; upgrading the `pytest` module fixes the conflict.
-pytest = "*"
-
-# Building the `tomli` module, which is in the dependency tree of `hatchling`,
-# requires a newer `flit-core` module than found in the used Nixpkgs snapshot.
-flit-core = "*"
-
-# Building `dotty-dict` >= 1.3.1 requires the `poetry-core` module, and the
-# version of that module provided by the used Nixpkgs snapshot cannot be built
-# on Darwin due to the regex compatibility issue in the old Nixpkgs code
-# (https://github.com/NixOS/nix/issues/4758).
-poetry-core = "*"
-
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

util/nix/sources.json

@@ -17,10 +17,10 @@
         "homepage": "",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c0e881852006b132236cbf0301bd1939bb50867e",
+        "rev": "98b00b6947a9214381112bdb6f89c25498db4959",
-        "sha256": "0fy7z7yxk5n7yslsvx5cyc6h21qwi4bhxf3awhirniszlbvaazy2",
+        "sha256": "1m6dm144mbm56n9293m26f46bjrklknyr4q4kzvxkiv036ijma98",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/c0e881852006b132236cbf0301bd1939bb50867e.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/98b00b6947a9214381112bdb6f89c25498db4959.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "poetry2nix": {
@@ -29,10 +29,10 @@
         "homepage": "",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "11c0df8e348c0f169cd73a2e3d63f65c92baf666",
+        "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609",
-        "sha256": "0i3wbp2p0x6bpj07sqpvkbx4lvjm0irvpmv2bjqx8k02mpjm7dg2",
+        "sha256": "1jfrangw0xb5b8sdkimc550p3m98zhpb1fayahnr7crg74as4qyq",
         "type": "tarball",
-        "url": "https://github.com/nix-community/poetry2nix/archive/11c0df8e348c0f169cd73a2e3d63f65c92baf666.tar.gz",
+        "url": "https://github.com/nix-community/poetry2nix/archive/3c92540611f42d3fb2d0d084a6c694cd6544b609.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

util/nix/sources.nix

@@ -10,29 +10,50 @@ let
     let
       name' = sanitizeName name + "-src";
     in
-      if spec.builtin or true then
+    if spec.builtin or true then
-        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      builtins_fetchurl { inherit (spec) url sha256; name = name'; }
-      else
+    else
-        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
+      pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
 
   fetch_tarball = pkgs: name: spec:
     let
       name' = sanitizeName name + "-src";
     in
-      if spec.builtin or true then
+    if spec.builtin or true then
-        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
-      else
+    else
-        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
+      pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
 
   fetch_git = name: spec:
     let
       ref =
-        if spec ? ref then spec.ref else
+        spec.ref or (
           if spec ? branch then "refs/heads/${spec.branch}" else
-            if spec ? tag then "refs/tags/${spec.tag}" else
+          if spec ? tag then "refs/tags/${spec.tag}" else
-              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+          abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"
+        );
+      submodules = spec.submodules or false;
+      submoduleArg =
+        let
+          nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
+          emptyArgWithWarning =
+            if submodules
+            then
+              builtins.trace
+                (
+                  "The niv input \"${name}\" uses submodules "
+                  + "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
+                  + "does not support them"
+                )
+                { }
+            else { };
+        in
+        if nixSupportsSubmodules
+        then { inherit submodules; }
+        else emptyArgWithWarning;
     in
-      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
+    builtins.fetchGit
+      ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);
 
   fetch_local = spec: spec.path;
 
@@ -66,16 +87,16 @@ let
       hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
       hasThisAsNixpkgsPath = <nixpkgs> == ./.;
     in
-      if builtins.hasAttr "nixpkgs" sources
+    if builtins.hasAttr "nixpkgs" sources
-      then sourcesNixpkgs
+    then sourcesNixpkgs
-      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+    else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
-        import <nixpkgs> {}
+      import <nixpkgs> { }
-      else
+    else
-        abort
+      abort
-          ''
+        ''
-            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+          Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
-            add a package called "nixpkgs" to your sources.json.
+          add a package called "nixpkgs" to your sources.json.
-          '';
+        '';
 
   # The actual fetching function.
   fetch = pkgs: name: spec:
@@ -95,13 +116,13 @@ let
   # the path directly as opposed to the fetched source.
   replace = name: drv:
     let
-      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name;
       ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
     in
-      if ersatz == "" then drv else
+    if ersatz == "" then drv else
-        # this turns the string into an actual Nix path (for both absolute and
+      # this turns the string into an actual Nix path (for both absolute and
-        # relative paths)
+      # relative paths)
-        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
+    if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
 
   # Ports of functions for older nix versions
 
@@ -112,7 +133,7 @@ let
   );
 
   # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
-  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+  range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1);
 
   # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
   stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
@@ -123,43 +144,46 @@ let
   concatStrings = builtins.concatStringsSep "";
 
   # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
-  optionalAttrs = cond: as: if cond then as else {};
+  optionalAttrs = cond: as: if cond then as else { };
 
   # fetchTarball version that is compatible between all the versions of Nix
   builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchTarball;
     in
-      if lessThan nixVersion "1.12" then
+    if lessThan nixVersion "1.12" then
-        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
-      else
+    else
-        fetchTarball attrs;
+      fetchTarball attrs;
 
   # fetchurl version that is compatible between all the versions of Nix
   builtins_fetchurl = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchurl;
     in
-      if lessThan nixVersion "1.12" then
+    if lessThan nixVersion "1.12" then
-        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
-      else
+    else
-        fetchurl attrs;
+      fetchurl attrs;
 
   # Create the final "sources" from the config
   mkSources = config:
-    mapAttrs (
+    mapAttrs
-      name: spec:
+      (
-        if builtins.hasAttr "outPath" spec
+        name: spec:
-        then abort
+          if builtins.hasAttr "outPath" spec
-          "The values in sources.json should not have an 'outPath' attribute"
+          then
-        else
+            abort
-          spec // { outPath = replace name (fetch config.pkgs name spec); }
+              "The values in sources.json should not have an 'outPath' attribute"
-    ) config.sources;
+          else
+            spec // { outPath = replace name (fetch config.pkgs name spec); }
+      )
+      config.sources;
 
   # The "config" used by the fetchers
   mkConfig =
     { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
-    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile)
     , system ? builtins.currentSystem
     , pkgs ? mkPkgs sources system
     }: rec {
@@ -171,4 +195,4 @@ let
     };
 
 in
-mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }
+mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); }