Merge main features from denote-search into Denote

denote.el

@@ -2539,17 +2539,21 @@ pass it to `denote-directory-files'."
     (denote-retrieve-groups-xref-query query files-matching-regexp))
    #'string-collate-lessp))
 
-(defun denote-retrieve-xref-alist (query &optional files-matching-regexp)
+(defun denote-retrieve-xref-alist (query &optional files)
   "Return xref alist of absolute file paths with location of matches for QUERY.
-With optional FILES-MATCHING-REGEXP, limit the list of files
-accordingly (per `denote-directory-files').
+Optional FILES can be a list of files to search for.  It can also be a
+regular expression, which means to use the text files in
+`denote-directory' that match that regexp.
 
-At all times limit the search to text files."
+If FILES is not given, use all text files as returned by
+`denote-directory-files'."
   (let ((xref-file-name-display 'abs))
     (xref--analyze
      (xref-matches-in-files
       query
-      (denote-directory-files files-matching-regexp :omit-current :text-only)))))
+      (if (and files (listp files))
+          files
+        (denote-directory-files files denote-query--omit-current :text-only))))))
 
 ;;;; New note
 
@@ -5100,25 +5104,89 @@ file's title.  This has the same meaning as in `denote-link'."
   'denote-query-mode
   "4.0.0")
 
+(declare-function outline-cycle "outline" (&optional event))
+(declare-function outline-cycle-buffer "outline" (&optional level))
+(declare-function outline-next-heading "outline" ())
+(declare-function outline-previous-heading "outline" ())
+
+(defvar denote-query-mode-map
+  (let ((map (make-sparse-keymap)))
+    (define-key map "a" #'outline-cycle-buffer)
+    (define-key map "f" #'denote-grep-refine)
+    (define-key map "k" #'outline-previous-heading)
+    (define-key map "j" #'outline-next-heading)
+    (define-key map "o" #'delete-other-windows)
+    (define-key map "s" #'denote-grep)
+    (define-key map "v" #'outline-cycle)
+    (define-key map "x" #'denote-grep-exclude-files)
+    (define-key map "i" #'denote-grep-only-include-files)
+    (define-key map "l" #'recenter-current-error)
+    (define-key map "X" #'denote-grep-exclude-files-with-keywords)
+    (define-key map "I" #'denote-grep-only-include-files-with-keywords)
+    (define-key map "G" #'denote-grep-clear-all-filters)
+    map)
+  "Keymap for buffers generated by `denote-make-links-buffer'.")
+
 (define-derived-mode denote-query-mode xref--xref-buffer-mode "Denote Query"
   "Major mode for queries found in the variable `denote-directory'.
 This is used by the command `denote-backlinks' and all links created by
 the `denote-query' command, among others."
-  :interactive nil)
+  :interactive nil
+  (require 'outline)
+  (setq-local outline-minor-mode-use-buttons 'in-margins)
+  (outline-minor-mode))
 
 (define-obsolete-function-alias
   'denote-link--prepare-backlinks
   'denote-make-links-buffer
   "4.0.0")
 
+(defcustom denote-query-format-heading-function #'identity
+  "Function used to construct headings for files matched by a query.
+
+It is called with a single argument, the path to the note file, and it
+should always return a string."
+  :type 'function)
+
+(defcustom denote-query-untitled-string "[Untitled]"
+  "String to use as heading for untitled notes in links' buffer.
+
+Used only by `denote-query-extract-title'."
+  :type 'string)
+
+(defvar denote-query--last-files nil
+  "Variable holding files matched by last call to `denote-make-links-buffer'.")
+
+(defvar denote-query--last-query nil
+  "Variable holding query string for last call to `denote-make-links-buffer'.")
+
+(defvar denote-query--omit-current t
+  "Variable governing whether query should omit current file.")
+
+(defun denote-query-extract-title (file)
+  "Extract note title from FILE front matter.
+
+When no title is found, return title found in FILE name.
+
+When that doesn't work, return `denote-grep-untitled-string'.
+
+Intended to be used as `denote-query-format-heading-function'."
+  (let ((title
+         (denote-retrieve-title-or-filename
+          file
+          (denote-filetype-heuristics file))))
+    (if (and (stringp title) (not (string-blank-p title)))
+        title
+      denote-query-untitled-string)))
+
 ;; NOTE 2025-03-24: The `&rest' is there because we used to have an
 ;; extra SHOW-CONTEXT parameter.  This way we do not break anybody's
 ;; code, even if we slightly modify the behaviour.
-(defun denote-make-links-buffer (query &optional files-matching-regexp buffer-name display-buffer-action &rest _)
+(defun denote-make-links-buffer (query &optional files buffer-name display-buffer-action &rest _)
   "Create links' buffer called BUFFER-NAME for QUERY.
 
-With optional FILES-MATCHING-REGEXP, limit the list of files
-accordingly (per `denote-directory-files').
+Optional FILES can be a list of files to search for.  It can also be a
+regexp, which limits the files accordingly per `denote-directory-files'.
 
 Optional DISPLAY-BUFFER-ACTION is a `display-buffer' action and
 concomitant alist, such as `denote-backlinks-display-buffer-action'."
@@ -5130,10 +5198,29 @@ concomitant alist, such as `denote-backlinks-display-buffer-action'."
          ;; suitable function and the results would be automatically
          ;; in relative form, but eventually notes may not be all
          ;; under a common directory (or project).
-         (xref-alist (denote-retrieve-xref-alist query files-matching-regexp))
+         (xref-alist (denote-retrieve-xref-alist query files))
          (dir (denote-directory)))
     (unless xref-alist
       (error "No matches for query `%s'" query))
+    ;; Update internal variables
+    (setq denote-query--last-files nil)
+    (setq denote-query--last-query query)
+    (dolist (x xref-alist)
+      (let* ((file-xref (car x))
+             (file
+              ;; NOTE: Unfortunately, the car of the xref construct is
+              ;; not reliable; sometimes it's absolute, sometimes it
+              ;; is not
+              (if (file-name-absolute-p file-xref)
+                  file-xref
+                (xref-location-group
+                 (xref-match-item-location (car (last x)))))))
+        ;; Add to current set of files
+        (push file denote-query--last-files)
+        ;; Format heading
+        (setf (car x) (funcall denote-query-format-heading-function file))))
+    (delete-dups denote-query--last-files)
+    ;; Insert results
     (with-current-buffer (get-buffer-create buffer)
       (erase-buffer)
       (denote-query-mode)
@@ -5150,13 +5237,178 @@ concomitant alist, such as `denote-backlinks-display-buffer-action'."
       (setq-local revert-buffer-function
                   (lambda (_ignore-auto _noconfirm)
                     (when-let* ((buffer-file-name file))
-                      (denote-make-links-buffer query files-matching-regexp buffer-name display-buffer-action)))))
+                      (denote-make-links-buffer query files buffer-name display-buffer-action)))))
     (display-buffer buffer display-buffer-action)))
 
 (defvar denote-query-links-buffer-function #'denote-make-links-buffer
   "Function to make an Xref buffer showing query link results.
 It accepts the same arguments as `denote-make-links-buffer'.")
 
+;;;;;; Additional features for searching file contents
+
+(defvar denote-grep-query-history nil
+  "Minibuffer history for `denote-grep' commands.")
+
+(defvar denote-grep-file-regexp-history nil
+  "Minibuffer history for `denote-grep' commands asking for a file regexp.")
+
+(defun denote-grep-query-prompt (&optional type)
+  "Prompt for a search query in the minibuffer.
+
+The prompt assumes a search in all files, unless TYPE is non-nil.
+
+For now, the only recognized value for TYPE is :focused (for a focused
+search, see `denote-grep-refine').
+
+TYPE only affects the prompt, not the returned value.
+
+Returned value is a list in order to be used in an `interactive' spec."
+  (list (read-string
+         (cond ((eq type :focused)
+                "Search (only files matched last): ")
+               (t "Search (all Denote files): "))
+         nil 'denote-grep-query-history)))
+
+(defun denote-grep-file-regexp-prompt (&optional include)
+  "Prompt for a file regexp in the minibuffer.
+
+The prompt assumes the user wants to exclude files, unless INCLUDE is
+non-nil.
+
+Returned value is a list in order to be used in an `interactive' spec."
+  (list (read-string
+         (if (not include)
+             "Exclude file names matching: "
+           "Only include file names matching: ")
+         nil 'denote-grep-file-regexp-history)))
+
+(defun denote-grep-keywords-prompt (&optional include)
+  "Prompt for keywords to filter in the minibuffer, with completion.
+
+Keywords are read using `completing-read-multiple'.
+
+The prompt assumes the user wants to exclude the keywords, unless
+INCLUDE is non-nil.
+
+Returned value is a list in order to be used in an `interactive' spec."
+  (list
+   (delete-dups
+    (completing-read-multiple
+     (if (not include)
+         "Exclude files with keywords: "
+       "Only include files with keywords: ")
+     (denote-keywords) nil t nil 'denote-keyword-history))))
+
+;;;###autoload
+(defun denote-grep (query)
+  "Search QUERY in the content of Denote files.
+QUERY should be a regular expression accepted by `xref-search-program'.
+
+The files to search for are those returned by `denote-directory-files'
+with a non-nil TEXT-ONLY argument.
+
+Results are put in a buffer which allows folding and further
+filtering (see the manual for details).
+
+You can insert a link to a grep search in any note by using the command
+`denote-query-contents-link'."
+  (interactive (denote-grep-query-prompt))
+  (let (denote-query--omit-current)
+    (denote-make-links-buffer query)))
+
+(defun denote-grep-refine (query)
+  "Search QUERY in the content of files which matched the last search.
+\"Last search\" here means any call to `denote-backlinks' and all links
+created by the `denote-query' command.
+
+QUERY should be regular expression."
+  (interactive (denote-grep-query-prompt :focused))
+  (denote-make-links-buffer
+   query denote-query--last-files
+   nil '(display-buffer-same-window))
+  (message "Searching `%s' in files matched previously" query))
+
+(defun denote-grep-exclude-files (regexp)
+  "Exclude files whose name matches REGEXP from current search buffer.
+
+This is useful even if you don't know regular expressions, given the
+Denote file-naming scheme.  For instance, to exclude notes with the
+keyword \"philosophy\" from current search buffer, type
+‘\\<denote-query-mode-map>\\[denote-grep-exclude-files] _philosophy
+RET’.
+
+Internally, this works by generating a new call to
+`denote-make-links-buffer' with the same QUERY as the last one, but with
+a set of files gotten from checking REGEXP against last matched files.
+
+When called from Lisp, REGEXP can be a list; in that case, it should be
+a list of fixed strings (NOT regexps) to check against last matched
+files.  Files that match any of the strings get excluded.  Internally,
+the list is processed using `regexp-opt'.  For an example of this usage,
+see `denote-grep-exclude-files-with-keywords'."
+  (interactive (denote-grep-file-regexp-prompt))
+  (let (final-files)
+    (dolist (file denote-query--last-files)
+      (unless (string-match
+               ;; Support list of strings as REGEXP
+               (if (listp regexp)
+                   (regexp-opt regexp)
+                 regexp)
+               file)
+        (push file final-files)))
+    (if final-files
+        (denote-make-links-buffer denote-query--last-query final-files)
+      (user-error "No remaining files when applying that filter"))
+    (message "Excluding files matching `%s'" regexp)))
+
+(defun denote-grep-only-include-files (regexp)
+  "Exclude file names not matching REGEXP from current query buffer.
+
+See `denote-grep-exclude-files' for details, including the behaviour
+when REGEXP is a list."
+  (interactive (denote-grep-file-regexp-prompt :include))
+  (let (final-files)
+    (dolist (file denote-query--last-files)
+      (when (string-match
+             ;; Support list of strings as REGEXP
+             (if (listp regexp)
+                 (regexp-opt regexp)
+               regexp)
+             file)
+        (push file final-files)))
+    (if final-files
+        (denote-make-links-buffer denote-query--last-query final-files)
+      (user-error "No remaining files when applying that filter"))
+    (message "Only including files matching `%s'" regexp)))
+
+(defun denote-grep-exclude-files-with-keywords (keywords)
+  "Exclude files with KEYWORDS from current query buffer.
+
+KEYWORDS should be a list of keywords (without underscore).
+
+Interactively, KEYWORDS are read from the minibuffer using
+`completing-read-multiple', which see."
+  (interactive (denote-grep-keywords-prompt))
+  (denote-grep-exclude-files
+   (mapcar (lambda (kw) (concat "_" kw)) keywords)))
+
+(defun denote-grep-only-include-files-with-keywords (keywords)
+  "Exclude files without KEYWORDS from current query buffer.
+
+See `denote-grep-exclude-files-with-keywords' for details."
+  (interactive (denote-grep-keywords-prompt :include))
+  (denote-grep-only-include-files
+   (mapcar (lambda (kw) (concat "_" kw)) keywords)))
+
+(defun denote-grep-clear-all-filters ()
+  "Run the last search with the full set of files in `denote-directory'.
+
+This effectively gets ride of any interactive filter applied (by the
+means of e.g. `denote-grep-exclude-files')."
+  (interactive)
+  (denote-make-links-buffer denote-query--last-query)
+  (message "Cleared all filters"))
+
 ;;;;;; Backlinks
 
 (defun denote--backlinks-get-buffer-name (file id)