Revert "all: purge a bunch of unused packages"

This reverts some of commit 99b751f321.

flake.nix

@@ -324,7 +324,6 @@
           intiface-engine = upkgs.callPackage ./pkgs/intiface-engine.nix { };
           flake-warn =
             spkgs.callPackage ./pkgs/flake-warn.nix { inherit spkgs; };
-          #kurinto = spkgs.callPackage ./pkgs/kurinto.nix {};
           gen-patches =
             spkgs.callPackage ./bins/gen-patches.nix { inherit spkgs; };
           yarr = spkgs.callPackage ./pkgs/yarr.nix {
@@ -340,12 +339,27 @@
           rtlamr2mqtt = spkgs.python3Packages.callPackage ./pkgs/rtlamr2mqtt.nix {
             inherit spkgs;
           };
+          kobuddy = upkgs.python3Packages.callPackage ./pkgs/kobuddy.nix {
+            inherit upkgs;
+          };
+          bandcamp-downloader = upkgs.python3Packages.callPackage ./pkgs/bandcamp-downloader.nix {
+            inherit upkgs;
+          };
+          ghexport = upkgs.python3Packages.callPackage ./pkgs/ghexport.nix {
+            inherit upkgs;
+          };
+          hpi =
+            upkgs.python3Packages.callPackage ./pkgs/hpi.nix { inherit upkgs; };
           openevse =
             upkgs.python312Packages.callPackage ./pkgs/openevse.nix { inherit upkgs; };
           ble-serial =
             upkgs.python312Packages.callPackage ./pkgs/ble-serial.nix { inherit upkgs; };
+          promnesia = upkgs.python3Packages.callPackage ./pkgs/promnesia.nix {
+            inherit upkgs;
+          };
           sliding-sync =
             spkgs.callPackage ./pkgs/sliding-sync.nix { inherit spkgs; };
+          golink = spkgs.callPackage ./pkgs/golink.nix { inherit spkgs; };
           gokrazy = upkgs.callPackage ./pkgs/gokrazy.nix { inherit upkgs; };
           gosignify = spkgs.callPackage ./pkgs/gosignify.nix { inherit spkgs; };
           gotosocial =
@@ -353,6 +367,9 @@
           zutty = upkgs.callPackage ./pkgs/zutty.nix {
             inherit upkgs;
           };
+          mvoice = upkgs.callPackage ./pkgs/mvoice.nix {
+            inherit upkgs;
+          };
           inherit (xintray.packages.${system}) xintray;
           inherit (beyt.packages.${system}) beyt;
           inherit (tsvnstat.packages.${system}) tsvnstat;

gui/default.nix

@@ -16,7 +16,6 @@ let
   rpr =
     pkgs.writeScriptBin "rpr"
       (import ../bins/rpr.nix { inherit (pkgs) hut gh tea; });
-
   editorScript = pkgs.writeShellScriptBin "emacseditor" ''
     if [ -z "$1" ]; then
       exec ${myEmacs}/bin/emacsclient --create-frame --alternate-editor ${myEmacs}/bin/emacs
@@ -24,7 +23,27 @@ let
       exec ${myEmacs}/bin/emacsclient --alternate-editor ${myEmacs}/bin/emacs "$@"
     fi
   '';
+  promnesia =
+    pkgs.python3Packages.callPackage ../pkgs/promnesia.nix { inherit pkgs; };
+  hpi = pkgs.python3Packages.callPackage ../pkgs/hpi.nix { inherit pkgs; };
+  promnesiaService = {
+    promnesia = {
+      description = "Service for promnesia.server";
+      wantedBy = [ "graphical-session.target" ];
+      partOf = [ "graphical-session.target" ];
+      after = [ "graphical-session.target" ];
+      script = ''
+        ${promnesia}/bin/promnesia serve
+      '';
+    };
+  };
   jobs = [
+    {
+      name = "promnesia-index";
+      script = "${promnesia}/bin/promnesia index";
+      startAt = "*:0/5";
+      path = [ promnesia hpi ];
+    }
   ];
   fontSet = with pkgs; [
     go-font
@@ -105,10 +124,12 @@ with lib; {
           exiftool
           go-font
           govulncheck
+          hpi
           keepassxc
           mpv
           pcsctools
           plan9port
+          promnesia
           rage
           rpr
           traygent
@@ -126,7 +147,8 @@ with lib; {
       } // firefox.programs;
 
       systemd.user.services =
-        lib.listToAttrs (builtins.map xinlib.jobToUserService jobs);
+        (lib.listToAttrs (builtins.map xinlib.jobToUserService jobs))
+        // promnesiaService;
       security.rtkit.enable = true;
     })
     (mkIf config.pipewire.enable {

hosts/europa/default.nix

@@ -390,7 +390,21 @@ in
       zig
 
       (callPackage ../../pkgs/ttfs.nix { })
+      (callPackage ../../pkgs/kobuddy.nix {
+        inherit pkgs;
+        inherit
+          (pkgs.python39Packages)
+          buildPythonPackage
+          fetchPypi
+          setuptools-scm
+          pytz
+          banal
+          sqlalchemy
+          alembic
+          ;
+      })
       (callPackage ../../pkgs/gokrazy.nix { })
+      (callPackage ../../pkgs/mvoice.nix { })
       (callPackage ../../pkgs/zutty.nix { })
       #(python3Packages.callPackage ../../pkgs/watchmap.nix { })
       (python3Packages.callPackage ../../pkgs/ble-serial.nix { })

pkgs/cachew.nix

@@ -0,0 +1,22 @@
+{ buildPythonPackage
+, fetchPypi
+, setuptools-scm
+, appdirs
+, sqlalchemy
+, ...
+}:
+buildPythonPackage rec {
+  pname = "cachew";
+  version = "0.11.0";
+
+  nativeBuildInputs = [ setuptools-scm ];
+
+  doCheck = true;
+
+  propagatedBuildInputs = [ appdirs sqlalchemy ];
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-4qjgvffInKRpKST9xbwwC2+m8h3ups0ZePyJLUU+KhA=";
+  };
+}

pkgs/ghexport.nix

@@ -0,0 +1,31 @@
+{ buildPythonPackage
+, setuptools-scm
+, fetchFromGitHub
+, PyGithub
+, pytz
+, ...
+}:
+buildPythonPackage rec {
+  pname = "ghexport";
+  version = "20220828";
+
+  nativeBuildInputs = [ setuptools-scm ];
+  propagatedBuildInputs = [ PyGithub pytz ];
+
+  doCheck = true;
+
+  buildInputs = [ ];
+
+  preCheck = ''
+    export HOME=$(mktemp -d)
+  '';
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  src = fetchFromGitHub {
+    owner = "karlicoss";
+    repo = pname;
+    rev = "e7704dc5b984731a53e74cbcadcbc3dd9c3024cd";
+    hash = "sha256-m/iqeBvCXHlN7GsNW6A2AX1g+ZaH3W62+Ulcfgup0KQ=";
+  };
+}

pkgs/hpi.nix

@@ -0,0 +1,67 @@
+{ pkgs
+, buildPythonPackage
+, setuptools-scm
+, pytest
+, fetchPypi
+, appdirs
+, click
+, decorator
+, geopy
+, logzero
+, lxml
+, more-itertools
+, mypy
+, orjson
+, pandas
+, pytz
+, simplejson
+, ...
+}:
+with pkgs; let
+  orgparse = pkgs.python3Packages.callPackage ./orgparse.nix { inherit pkgs; };
+  kobuddy = pkgs.python3Packages.callPackage ./kobuddy.nix { inherit pkgs; };
+  ghexport = pkgs.python3Packages.callPackage ./ghexport.nix { inherit pkgs; };
+in
+buildPythonPackage rec {
+  pname = "HPI";
+  version = "0.3.20230207";
+
+  nativeBuildInputs = [ setuptools-scm ];
+  propagatedBuildInputs = [
+    appdirs
+    click
+    decorator
+    geopy
+    kobuddy
+    logzero
+    lxml
+    ghexport
+    more-itertools
+    mypy
+    orgparse
+    orjson
+    pandas
+    pytest
+    pytz
+    simplejson
+  ];
+
+  doCheck = true;
+
+  buildInputs = [ mypy kobuddy ];
+
+  makeWrapperArgs = [
+    # Add the installed directories to the python path so the daemon can find them
+    "--prefix PYTHONPATH : ${python3.pkgs.makePythonPath propagatedBuildInputs}"
+    "--prefix PYTHONPATH : $out/lib/${python3.libPrefix}/site-packages"
+  ];
+
+  preCheck = ''
+    export HOME=$(mktemp -d)
+  '';
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-i3C1Lmj6K48zVG960uv1epQm38qQnxalwy8kHnLTZrE=";
+  };
+}

pkgs/kobuddy.diff

@@ -0,0 +1,183 @@
+diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
+index 68bf5db..92d375e 100644
+--- a/.github/workflows/build.yml
++++ b/.github/workflows/build.yml
+@@ -42,6 +42,9 @@ jobs:
+         run: |
+           sudo apt-get -qq update
+           pip install -e ".[dev]"
++
++      - uses: mxschmitt/action-tmate@v3
++
+       - name: Run SQLite tests
+         env:
+           DATABASE_URL: "sqlite:///:memory:"
+diff --git a/dataset/database.py b/dataset/database.py
+index d8a07ad..4bc31fd 100644
+--- a/dataset/database.py
++++ b/dataset/database.py
+@@ -106,7 +106,7 @@ class Database(object):
+     @property
+     def metadata(self):
+         """Return a SQLAlchemy schema cache object."""
+-        return MetaData(schema=self.schema, bind=self.executable)
++        return MetaData(schema=self.schema)
+ 
+     @property
+     def in_transaction(self):
+@@ -127,6 +127,8 @@ class Database(object):
+         """
+         if not hasattr(self.local, "tx"):
+             self.local.tx = []
++        if self.executable.in_transaction():
++            self.executable.commit()
+         self.local.tx.append(self.executable.begin())
+ 
+     def commit(self):
+diff --git a/dataset/table.py b/dataset/table.py
+index 08b806b..2f27060 100644
+--- a/dataset/table.py
++++ b/dataset/table.py
+@@ -116,7 +116,12 @@ class Table(object):
+         Returns the inserted row's primary key.
+         """
+         row = self._sync_columns(row, ensure, types=types)
+-        res = self.db.executable.execute(self.table.insert(row))
++        res = self.db.executable.execute(self.table.insert(), row)
++        # SQLAlchemy 2.0.0b1 removes auto commit
++        if hasattr(self.db.local, "tx") and self.db.local.tx:
++            pass
++        else:
++            self.db.executable.commit()
+         if len(res.inserted_primary_key) > 0:
+             return res.inserted_primary_key[0]
+         return True
+@@ -181,7 +186,8 @@ class Table(object):
+             # Insert when chunk_size is fulfilled or this is the last row
+             if len(chunk) == chunk_size or index == len(rows) - 1:
+                 chunk = pad_chunk_columns(chunk, columns)
+-                self.table.insert().execute(chunk)
++                with self.db.engine.begin() as conn:
++                    conn.execute(self.table.insert(), chunk)
+                 chunk = []
+ 
+     def update(self, row, keys, ensure=None, types=None, return_count=False):
+@@ -206,7 +212,7 @@ class Table(object):
+         clause = self._args_to_clause(args)
+         if not len(row):
+             return self.count(clause)
+-        stmt = self.table.update(whereclause=clause, values=row)
++        stmt = self.table.update().where(clause).values(row)
+         rp = self.db.executable.execute(stmt)
+         if rp.supports_sane_rowcount():
+             return rp.rowcount
+@@ -241,10 +247,9 @@ class Table(object):
+             # Update when chunk_size is fulfilled or this is the last row
+             if len(chunk) == chunk_size or index == len(rows) - 1:
+                 cl = [self.table.c[k] == bindparam("_%s" % k) for k in keys]
+-                stmt = self.table.update(
+-                    whereclause=and_(True, *cl),
+-                    values={col: bindparam(col, required=False) for col in columns},
+-                )
++                stmt = self.table.update()\
++                                 .where(and_(True, *cl))\
++                                 .values({col: bindparam(col, required=False) for col in columns})
+                 self.db.executable.execute(stmt, chunk)
+                 chunk = []
+ 
+@@ -293,7 +298,7 @@ class Table(object):
+         if not self.exists:
+             return False
+         clause = self._args_to_clause(filters, clauses=clauses)
+-        stmt = self.table.delete(whereclause=clause)
++        stmt = self.table.delete().where(clause)
+         rp = self.db.executable.execute(stmt)
+         return rp.rowcount > 0
+ 
+@@ -303,7 +308,7 @@ class Table(object):
+             self._columns = None
+             try:
+                 self._table = SQLATable(
+-                    self.name, self.db.metadata, schema=self.db.schema, autoload=True
++                    self.name, self.db.metadata, schema=self.db.schema, autoload_with=self.db.engine,
+                 )
+             except NoSuchTableError:
+                 self._table = None
+@@ -625,7 +630,7 @@ class Table(object):
+ 
+         order_by = self._args_to_order_by(order_by)
+         args = self._args_to_clause(kwargs, clauses=_clauses)
+-        query = self.table.select(whereclause=args, limit=_limit, offset=_offset)
++        query = self.table.select().where(args).limit(_limit).offset(_offset)
+         if len(order_by):
+             query = query.order_by(*order_by)
+ 
+@@ -666,7 +671,7 @@ class Table(object):
+             return 0
+ 
+         args = self._args_to_clause(kwargs, clauses=_clauses)
+-        query = select([func.count()], whereclause=args)
++        query = select(func.count()).where(args)
+         query = query.select_from(self.table)
+         rp = self.db.executable.execute(query)
+         return rp.fetchone()[0]
+@@ -703,12 +708,10 @@ class Table(object):
+         if not len(columns):
+             return iter([])
+ 
+-        q = expression.select(
+-            columns,
+-            distinct=True,
+-            whereclause=clause,
+-            order_by=[c.asc() for c in columns],
+-        )
++        q = expression.select(*columns)\
++                      .distinct(True)\
++                      .where(clause)\
++                      .order_by(*(c.asc() for c in columns))
+         return self.db.query(q)
+ 
+     # Legacy methods for running find queries.
+diff --git a/setup.py b/setup.py
+index 0691373..fb794a4 100644
+--- a/setup.py
++++ b/setup.py
+@@ -30,7 +30,6 @@ setup(
+     include_package_data=False,
+     zip_safe=False,
+     install_requires=[
+-        "sqlalchemy >= 1.3.2, < 2.0.0",
+         "alembic >= 0.6.2",
+         "banal >= 1.0.1",
+     ],
+diff --git a/test/test_dataset.py b/test/test_dataset.py
+index f7c94eb..5861fbc 100644
+--- a/test/test_dataset.py
++++ b/test/test_dataset.py
+@@ -14,7 +14,10 @@ class DatabaseTestCase(unittest.TestCase):
+     def setUp(self):
+         self.db = connect()
+         self.tbl = self.db["weather"]
++        assert not self.db.has_table("weather")
+         self.tbl.insert_many(TEST_DATA)
++        # table is only created after insert statement
++        assert self.db.has_table("weather")
+ 
+     def tearDown(self):
+         for table in self.db.tables:
+@@ -83,7 +86,6 @@ class DatabaseTestCase(unittest.TestCase):
+     def test_create_table_shorthand1(self):
+         pid = "int_id"
+         table = self.db.get_table("foo5", pid)
+-        assert table.table.exists
+         assert len(table.table.columns) == 1, table.table.columns
+         assert pid in table.table.c, table.table.c
+ 
+@@ -98,7 +100,6 @@ class DatabaseTestCase(unittest.TestCase):
+         table = self.db.get_table(
+             "foo6", primary_id=pid, primary_type=self.db.types.string(255)
+         )
+-        assert table.table.exists
+         assert len(table.table.columns) == 1, table.table.columns
+         assert pid in table.table.c, table.table.c
+ 

pkgs/kobuddy.nix

@@ -0,0 +1,63 @@
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, fetchPypi
+, setuptools-scm
+, pytz
+, alembic
+, banal
+, sqlalchemy
+, ...
+}:
+let
+  myDataset = buildPythonPackage rec {
+    pname = "dataset";
+    version = "1.6.0";
+    format = "setuptools";
+
+    src = fetchFromGitHub {
+      owner = "pudo";
+      repo = pname;
+      rev = "0757b5010b600a66ed07fbb06a0e86c7bb0e09bc";
+      hash = "sha256-BfIGQvXKlsydV3p93/qLYtbVujTNWqWfMg16/aENHks=";
+    };
+
+    patches = [ ./kobuddy.diff ];
+
+    propagatedBuildInputs = [ alembic banal sqlalchemy ];
+
+    # checks attempt to import nonexistent module 'test.test' and fail
+    doCheck = false;
+
+    pythonImportsCheck = [ "dataset" ];
+
+    meta = with lib; {
+      description = "Toolkit for Python-based database access";
+      homepage = "https://dataset.readthedocs.io";
+      license = licenses.mit;
+      maintainers = with maintainers; [ xfnw ];
+    };
+  };
+in
+buildPythonPackage rec {
+  pname = "kobuddy";
+  version = "0.2.20221023";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-2Al1aDx9ymr0Pw+HC2S6mXkKvsDLhM1Oto+urr9i7BY=";
+  };
+
+  doCheck = true;
+
+  nativeBuildInputs = [ setuptools-scm ];
+
+  propagatedBuildInputs = [ myDataset pytz ];
+
+  meta = with lib; {
+    homepage = "https://github.com/karlicoss/promnesia";
+    description = "Another piece of your extended mind";
+    license = licenses.mit;
+    maintainers = with maintainers; [ qbit ];
+  };
+}

pkgs/orgparse.nix

@@ -0,0 +1,22 @@
+{ buildPythonPackage
+, setuptools-scm
+, pytest
+, fetchPypi
+, ...
+}:
+buildPythonPackage rec {
+  pname = "orgparse";
+  version = "0.3.2";
+
+  nativeBuildInputs = [ setuptools-scm ];
+  #propagatedBuildInputs = [ ];
+
+  nativeCheckInputs = [ pytest ];
+
+  doCheck = true;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-RRBQ55rLelHGXcmbkJXq5NUL1ZhUE1T552PLTL31mlU=";
+  };
+}

pkgs/promnesia.nix

@@ -0,0 +1,91 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, beautifulsoup4
+, fastapi
+, httptools
+, logzero
+, lxml
+, mistletoe
+, more-itertools
+, mypy
+, pytz
+, setuptools
+, tzlocal
+, urlextract
+, uvicorn
+, uvloop
+, watchfiles
+, websockets
+, setuptools-scm
+, starlette
+, pkgs
+, ...
+}:
+with pkgs; let
+  hpi = pkgs.python3Packages.callPackage ./hpi.nix { inherit pkgs; };
+  sqlcipher3 =
+    pkgs.python3Packages.callPackage ./sqlcipher3.nix { inherit pkgs; };
+  cachew = pkgs.python3Packages.callPackage ./cachew.nix { inherit pkgs; };
+  python-dotenv = pkgs.python3Packages.callPackage ./python-dotenv.nix { };
+in
+buildPythonPackage rec {
+  pname = "promnesia";
+  version = "1.1.20240810";
+
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "karlicoss";
+    repo = pname;
+    rev = "61f1c47992881f85748c8a184f1e0946bf69bb21";
+    hash = "sha256-vBYH2xKIWDcaQ5ee6aZYCBEkFtvv+OnFirq0WeyXBrQ=";
+  };
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  doCheck = true;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  # Optional
+  # bs4 lxml mistletoe logzero
+  propagatedBuildInputs = [
+    beautifulsoup4
+    cachew
+    fastapi
+    hpi
+    httptools
+    starlette
+    logzero
+    lxml
+    mistletoe
+    more-itertools
+    mypy
+    python-dotenv
+    pytz
+    setuptools
+    sqlcipher3
+    tzlocal
+    urlextract
+    uvicorn
+    uvloop
+    watchfiles
+    websockets
+  ];
+
+  makeWrapperArgs = [
+    # Add the installed directories to the python path so the daemon can find them
+    "--prefix PYTHONPATH : ${python3.pkgs.makePythonPath propagatedBuildInputs}"
+    "--prefix PYTHONPATH : $out/lib/${python3.libPrefix}/site-packages"
+  ];
+
+  meta = with lib; {
+    homepage = "https://github.com/karlicoss/promnesia";
+    description = "Another piece of your extended mind";
+    license = licenses.mit;
+    maintainers = with maintainers; [ qbit ];
+  };
+}

pkgs/python-dotenv.nix

@@ -0,0 +1,36 @@
+{ lib
+, buildPythonPackage
+, click
+, fetchPypi
+, ipython
+, mock
+, pytestCheckHook
+, pythonOlder
+, sh
+,
+}:
+buildPythonPackage rec {
+  pname = "python-dotenv";
+  version = "0.21.0";
+  disabled = pythonOlder "3.5";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-t30IJ0Y549NBRd+mxwCOZt8PBLe+enX9DVKSwZHXkEU=";
+  };
+
+  propagatedBuildInputs = [ click ];
+
+  nativeCheckInputs = [ ipython mock pytestCheckHook sh ];
+
+  disabledTests = [ "cli" ];
+
+  pythonImportsCheck = [ "dotenv" ];
+
+  meta = with lib; {
+    description = "Add .env support to your django/flask apps in development and deployments";
+    homepage = "https://github.com/theskumar/python-dotenv";
+    license = licenses.bsdOriginal;
+    maintainers = with maintainers; [ erikarvstedt ];
+  };
+}

pkgs/setup-hook.sh

@@ -0,0 +1,9 @@
+addHarepath() {
+	for haredir in third-party stdlib; do
+		if [[ -d "$1/src/hare/$haredir" ]]; then
+			addToSearchPath HAREPATH "$1/src/hare/$haredir"
+		fi
+	done
+}
+
+addEnvHooks "$hostOffset" addHarepath

pkgs/sqlcipher3.nix

@@ -0,0 +1,20 @@
+{ buildPythonPackage
+, setuptools-scm
+, sqlcipher
+, fetchPypi
+, ...
+}:
+buildPythonPackage rec {
+  pname = "sqlcipher3";
+  version = "0.5.0";
+
+  nativeBuildInputs = [ setuptools-scm ];
+  propagatedBuildInputs = [ sqlcipher ];
+
+  doCheck = true;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-+wa7UzaCWvIE6Obb/Ihema8UnFPr2P+HeDe1R4+iU+U=";
+  };
+}