vimPluginsUpdater: make development easier

`nix develop .#vimPluginsUpdater` now lets you enter a shell where you
can run `pkgs/applications/editors/vim/plugins/update.py` and
iteratively develop !

- removed `warn` warning from python by using `warning` instead
- `plugin2nix` was calling the same bit of code over and over thus
  slowing down the generator by a lot

maintainers/scripts/pluginupdate.py

@@ -142,7 +142,7 @@ class Repo:
         return loaded
 
     def prefetch(self, ref: Optional[str]) -> str:
-        print("Prefetching")
+        print("Prefetching %s", self.uri)
         loaded = self._prefetch(ref)
         return loaded["sha256"]
 
@@ -266,6 +266,7 @@ class PluginDesc:
 
     @staticmethod
     def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> "PluginDesc":
+        log.debug("Loading row %s", row)
         branch = row["branch"]
         repo = make_repo(row["repo"], branch.strip())
         repo.token = config.github_token
@@ -328,7 +329,7 @@ def load_plugins_from_csv(
 
 
 
-def run_nix_expr(expr, nixpkgs: str):
+def run_nix_expr(expr, nixpkgs: str, **args):
     '''
     :param expr nix expression to fetch current plugins
     :param nixpkgs Path towards a nixpkgs checkout
@@ -347,7 +348,7 @@ def run_nix_expr(expr, nixpkgs: str):
             nix_path,
         ]
         log.debug("Running command: %s", " ".join(cmd))
-        out = subprocess.check_output(cmd, timeout=90)
+        out = subprocess.check_output(cmd, **args)
         data = json.loads(out)
         return data
 
@@ -736,6 +737,7 @@ def rewrite_input(
     redirects: Redirects = {},
     append: List[PluginDesc] = [],
 ):
+    log.info("Rewriting input file %s", input_file)
     plugins = load_plugins_from_csv(
         config,
         input_file,
@@ -744,10 +746,14 @@ def rewrite_input(
     plugins.extend(append)
 
     if redirects:
+        log.debug("Dealing with deprecated plugins listed in %s", deprecated)
+
         cur_date_iso = datetime.now().strftime("%Y-%m-%d")
         with open(deprecated, "r") as f:
             deprecations = json.load(f)
+        # TODO parallelize this step
         for pdesc, new_repo in redirects.items():
+            log.info("Rewriting input file %s", input_file)
             new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias)
             old_plugin, _ = prefetch_plugin(pdesc)
             new_plugin, _ = prefetch_plugin(new_pdesc)
@@ -791,7 +797,7 @@ def update_plugins(editor: Editor, args):
     start_time = time.time()
     redirects = update()
     duration = time.time() - start_time
-    print(f"The plugin update took {duration}s.")
+    print(f"The plugin update took {duration:.2f}s.")
     editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, redirects)
 
     autocommit = not args.no_commit

pkgs/applications/editors/vim/plugins/update.py

@@ -40,7 +40,9 @@ ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe
 import pluginupdate
 import importlib
 from pluginupdate import run_nix_expr, PluginDesc
-import treesitter
+
+treesitter = importlib.import_module('nvim-treesitter.update')
+
 
 
 HEADER = (
@@ -54,14 +56,37 @@ class VimEditor(pluginupdate.Editor):
     nvim_treesitter_updated = False
 
     def generate_nix(
-        self, plugins: List[Tuple[PluginDesc, pluginupdate.Plugin]], outfile: str
+        self,
+        plugins: List[Tuple[PluginDesc, pluginupdate.Plugin]],
+        outfile: str
     ):
+        log.info("Generating nix code")
         sorted_plugins = sorted(plugins, key=lambda v: v[0].name.lower())
+        log.debug("Loading nvim-treesitter revision from nix...")
         nvim_treesitter_rev = pluginupdate.run_nix_expr(
-            "(import <localpkgs> { }).vimPlugins.nvim-treesitter.src.rev", self.nixpkgs
+            "(import <localpkgs> { }).vimPlugins.nvim-treesitter.src.rev",
+            self.nixpkgs,
+            timeout=10
         )
 
+        GET_PLUGINS_LUA = """
+        with import <localpkgs> {};
+        lib.attrNames lua51Packages"""
+        log.debug("Loading list of lua plugins...")
+        luaPlugins = run_nix_expr(GET_PLUGINS_LUA, self.nixpkgs, timeout=30)
+
+        def _isNeovimPlugin(plug: pluginupdate.Plugin) -> bool:
+            """
+            Whether it's a neovim-only plugin
+            We can check if it's available in lua packages
+            """
+            if plug.normalized_name in luaPlugins:
+                log.debug("%s is a neovim plugin", plug)
+                return True
+            return False
+
         with open(outfile, "w+") as f:
+            log.debug("Writing to %s", outfile)
             f.write(HEADER)
             f.write(
                 textwrap.dedent(
@@ -74,7 +99,7 @@ class VimEditor(pluginupdate.Editor):
                 )
             )
             for pdesc, plugin in sorted_plugins:
-                content = self.plugin2nix(pdesc, plugin)
+                content = self.plugin2nix(pdesc, plugin, _isNeovimPlugin(plugin))
                 f.write(content)
                 if (
                     plugin.name == "nvim-treesitter"
@@ -84,27 +109,10 @@ class VimEditor(pluginupdate.Editor):
             f.write("\n}\n")
         print(f"updated {outfile}")
 
-    def plugin2nix(self, pdesc: PluginDesc, plugin: pluginupdate.Plugin) -> str:
-        GET_PLUGINS_LUA = """
-        with import <localpkgs> {};
-        lib.attrNames lua51Packages"""
-        luaPlugins = run_nix_expr(GET_PLUGINS_LUA, self.nixpkgs)
+    def plugin2nix(self, pdesc: PluginDesc, plugin: pluginupdate.Plugin, isNeovim: bool) -> str:
 
         repo = pdesc.repo
 
-        def _isNeovimPlugin(plug: pluginupdate.Plugin) -> bool:
-            """
-            Whether it's a neovim-only plugin
-            We can check if it's available in lua packages
-            """
-            # global luaPlugins
-            if plug.normalized_name in luaPlugins:
-                log.debug("%s is a neovim plugin", plug)
-                return True
-            return False
-
-        isNeovim = _isNeovimPlugin(plugin)
-
         content = f"  {plugin.normalized_name} = "
         src_nix = repo.as_nix(plugin)
         content += """{buildFn} {{
@@ -159,8 +167,8 @@ class VimEditor(pluginupdate.Editor):
 def main():
     global luaPlugins
 
-    log.debug(f"Loading from {ROOT}/../get-plugins.nix")
-    with open(f"{ROOT}/../get-plugins.nix") as f:
+    log.debug(f"Loading from {ROOT}/get-plugins.nix")
+    with open(f"{ROOT}/get-plugins.nix") as f:
         GET_PLUGINS = f.read()
     editor = VimEditor(
         "vim", Path("pkgs/applications/editors/vim/plugins"), GET_PLUGINS

pkgs/applications/editors/vim/plugins/updater.nix

@@ -7,14 +7,14 @@
 , nurl
 
 # optional
-, vimPlugins
-, neovim
+, neovim-unwrapped
 }:
 buildPythonApplication {
-  format = "other";
   pname = "vim-plugins-updater";
   version = "0.1";
 
+  format = "other";
+
   nativeBuildInputs = [
     makeWrapper
     python3Packages.wrapPython
@@ -29,16 +29,18 @@ buildPythonApplication {
   installPhase = ''
     mkdir -p $out/bin $out/lib
     cp ${./update.py} $out/bin/vim-plugins-updater
-    cp ${./get-plugins.nix} $out/get-plugins.nix
-    cp ${./nvim-treesitter/update.py} $out/lib/treesitter.py
-    cp ${../../../../../maintainers/scripts/pluginupdate.py} $out/lib/pluginupdate.py
+    cp ${./get-plugins.nix} $out/bin/get-plugins.nix
 
     # wrap python scripts
     makeWrapperArgs+=( --prefix PATH : "${lib.makeBinPath [
-      nix nix-prefetch-git neovim nurl ]}" --prefix PYTHONPATH : "$out/lib" )
+      nix nix-prefetch-git neovim-unwrapped nurl ]}" --prefix PYTHONPATH : "${./.}:${../../../../../maintainers/scripts}" )
     wrapPythonPrograms
   '';
 
+  shellHook = ''
+    export PYTHONPATH=pkgs/applications/editors/vim/plugins:maintainers/scripts:$PYTHONPATH
+    '';
+
   meta.mainProgram = "vim-plugins-updater";
 }
 

pkgs/development/lua-modules/updater/updater.py

@@ -177,7 +177,7 @@ def generate_pkg_nix(plug: LuaPlugin):
                 "'version' and 'ref' will be ignored as the rockspec is hardcoded for package %s"
                 % plug.name
             )
-            log.warn(msg)
+            log.warning(msg)
 
         log.debug("Updating from rockspec %s", plug.rockspec)
         cmd.append(plug.rockspec)