project.el 21.8 KB
Newer Older
Dmitry Gutov's avatar
Dmitry Gutov committed
1 2
;;; project.el --- Operations on the current project  -*- lexical-binding: t; -*-

Paul Eggert's avatar
Paul Eggert committed
3
;; Copyright (C) 2015-2019 Free Software Foundation, Inc.
Dmitry Gutov's avatar
Dmitry Gutov committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17

;; This file is part of GNU Emacs.

;; GNU Emacs is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
18
;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
Dmitry Gutov's avatar
Dmitry Gutov committed
19 20 21 22

;;; Commentary:

;; This file contains generic infrastructure for dealing with
23 24
;; projects, some utility functions, and commands using that
;; infrastructure.
25
;;
26
;; The goal is to make it easier for Lisp programs to operate on the
27 28
;; current project, without having to know which package handles
;; detection of that project type, parsing its config files, etc.
29
;;
Dmitry Gutov's avatar
Dmitry Gutov committed
30 31 32 33 34
;; NOTE: The project API is still experimental and can change in major,
;; backward-incompatible ways.  Everyone is encouraged to try it, and
;; report to us any problems or use cases we hadn't anticipated, by
;; sending an email to emacs-devel, or `M-x report-emacs-bug'.
;;
35 36 37
;; Infrastructure:
;;
;; Function `project-current', to determine the current project
Dmitry Gutov's avatar
Dmitry Gutov committed
38
;; instance, and 5 (at the moment) generic functions that act on it.
39 40 41 42 43
;; This list is to be extended in future versions.
;;
;; Utils:
;;
;; `project-combine-directories' and `project-subtract-directories',
Paul Eggert's avatar
Paul Eggert committed
44
;; mainly for use in the abovementioned generics' implementations.
45 46 47 48 49 50 51 52
;;
;; Commands:
;;
;; `project-find-regexp' and `project-or-external-find-regexp' use the
;; current API, and thus will work in any project that has an adapter.

;;; TODO:

53 54 55 56 57 58
;; * Reliably cache the list of files in the project, probably using
;;   filenotify.el (if supported) to invalidate.  And avoiding caching
;;   if it's not available (manual cache invalidation is not nice).
;;
;; * Allow the backend to override the file-listing logic?  Maybe also
;;   to delegate file name completion to an external tool.
59 60 61 62 63 64 65 66 67
;;
;; * Build tool related functionality.  Start with a `project-build'
;;   command, which should provide completions on tasks to run, and
;;   maybe allow entering some additional arguments.  This might
;;   be handled better with a separate API, though.  Then we won't
;;   force every project backend to be aware of the build tool(s) the
;;   project is using.
;;
;; * Command to (re)build the tag files in all project roots.  To that
68 69 70 71
;;   end, we might need to add a way to provide file whitelist
;;   wildcards for each root to limit etags to certain files (in
;;   addition to the blacklist provided by ignores), and/or allow
;;   specifying additional tag regexps.
72 73 74 75 76 77 78 79
;;
;; * UI for the user to be able to pick the current project for the
;;   whole Emacs session, independent of the current directory.  Or,
;;   in the more advanced case, open a set of projects, and have some
;;   project-related commands to use them all.  E.g., have a command
;;   to search for a regexp across all open projects.  Provide a
;;   history of projects that were opened in the past (storing it as a
;;   list of directories should suffice).
80 81 82 83 84 85 86 87 88
;;
;; * Support for project-local variables: a UI to edit them, and a
;;   utility function to retrieve a value.  Probably useless without
;;   support in various built-in commands.  In the API, we might get
;;   away with only adding a `project-configuration-directory' method,
;;   defaulting to the project root the current file/buffer is in.
;;   And prompting otherwise.  How to best mix that with backends that
;;   want to set/provide certain variables themselves, is up for
;;   discussion.
Dmitry Gutov's avatar
Dmitry Gutov committed
89 90 91 92 93

;;; Code:

(require 'cl-generic)

94
(defvar project-find-functions (list #'project-try-vc)
Dmitry Gutov's avatar
Dmitry Gutov committed
95 96 97 98 99 100
  "Special hook to find the project containing a given directory.
Each functions on this hook is called in turn with one
argument (the directory) and should return either nil to mean
that it is not applicable, or a project instance.")

;;;###autoload
101 102 103
(defun project-current (&optional maybe-prompt dir)
  "Return the project instance in DIR or `default-directory'.
When no project found in DIR, and MAYBE-PROMPT is non-nil, ask
Dmitry Gutov's avatar
Dmitry Gutov committed
104 105 106
the user for a different directory to look in.  If that directory
is not a part of a detectable project either, return a
`transient' project instance rooted in it."
Dmitry Gutov's avatar
Dmitry Gutov committed
107
  (unless dir (setq dir default-directory))
108 109 110 111 112 113 114
  (let ((pr (project--find-in-directory dir)))
    (cond
     (pr)
     (maybe-prompt
      (setq dir (read-directory-name "Choose the project directory: " dir nil t)
            pr (project--find-in-directory dir))
      (unless pr
Dmitry Gutov's avatar
Dmitry Gutov committed
115 116
        (message "Using '%s' as a transient project root" dir)
        (setq pr (cons 'transient dir)))))
117 118 119
    pr))

(defun project--find-in-directory (dir)
Dmitry Gutov's avatar
Dmitry Gutov committed
120 121
  (run-hook-with-args-until-success 'project-find-functions dir))

122 123
(cl-defgeneric project-roots (project)
  "Return the list of directory roots of the current project.
124

125 126 127
Most often it's just one directory which contains the project
build file and everything else in the project.  But in more
advanced configurations, a project can span multiple directories.
128

129
The directory names should be absolute.")
130

131 132 133
;; FIXME: Add MODE argument, like in `ede-source-paths'?
(cl-defgeneric project-external-roots (_project)
  "Return the list of external roots for PROJECT.
134

135 136 137 138
It's the list of directories outside of the project that are
still related to it.  If the project deals with source code then,
depending on the languages used, this list should include the
headers search path, load path, class path, and so on.
139

140 141 142 143
The rule of thumb for whether to include a directory here, and
not in `project-roots', is whether its contents are meant to be
edited together with the rest of the project."
  nil)
Dmitry Gutov's avatar
Dmitry Gutov committed
144

145 146 147
(cl-defgeneric project-ignores (_project _dir)
  "Return the list of glob patterns to ignore inside DIR.
Patterns can match both regular files and directories.
Dmitry Gutov's avatar
Dmitry Gutov committed
148
To root an entry, start it with `./'.  To match directories only,
149
end it with `/'.  DIR must be one of `project-roots' or
150
`project-external-roots'."
Dmitry Gutov's avatar
Dmitry Gutov committed
151 152 153 154 155 156 157 158 159
  (require 'grep)
  (defvar grep-find-ignored-files)
  (nconc
   (mapcar
    (lambda (dir)
      (concat dir "/"))
    vc-directory-exclusion-list)
   grep-find-ignored-files))

160
(cl-defgeneric project-file-completion-table (project dirs)
Stephen Leake's avatar
Stephen Leake committed
161 162 163
  "Return a completion table for files in directories DIRS in PROJECT.
DIRS is a list of absolute directories; it should be some
subset of the project roots and external roots.
164

165 166
The default implementation delegates to `project-files'."
  (let ((all-files (project-files project dirs)))
167 168 169
    (lambda (string pred action)
      (cond
       ((eq action 'metadata)
170
        '(metadata . ((category . project-file))))
171
       (t
172
        (complete-with-action action all-files string pred))))))
Stephen Leake's avatar
Stephen Leake committed
173

Dmitry Gutov's avatar
Dmitry Gutov committed
174 175 176
(cl-defmethod project-roots ((project (head transient)))
  (list (cdr project)))

177 178 179
(cl-defgeneric project-files (project &optional dirs)
  "Return a list of files in directories DIRS in PROJECT.
DIRS is a list of absolute directories; it should be some
180 181 182 183 184 185 186
subset of the project roots and external roots.

The default implementation uses `find-program'.  PROJECT is used
to find the list of ignores for each directory."
  (require 'xref)
  (cl-mapcan
   (lambda (dir)
187 188
     (project--files-in-directory dir
                                  (project--dir-ignores project dir)))
189
   (or dirs (project-roots project))))
190

191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
(defun project--files-in-directory (dir ignores &optional files)
  (require 'find-dired)
  (defvar find-name-arg)
  (let ((command (format "%s %s %s -type f %s -print0"
                         find-program
                         dir
                         (xref--find-ignores-arguments
                          ignores
                          (expand-file-name dir))
                         (if files
                             (concat (shell-quote-argument "(")
                                     " " find-name-arg " "
                                     (mapconcat
                                      #'shell-quote-argument
                                      (split-string files)
                                      (concat " -o " find-name-arg " "))
                                     " "
                                     (shell-quote-argument ")"))"")
                         )))
    (split-string (shell-command-to-string command) "\0" t)))

212 213
(defgroup project-vc nil
  "Project implementation using the VC package."
Glenn Morris's avatar
Glenn Morris committed
214
  :version "25.1"
215 216 217
  :group 'tools)

(defcustom project-vc-ignores nil
Glenn Morris's avatar
Glenn Morris committed
218
  "List of patterns to include in `project-ignores'."
219 220 221
  :type '(repeat string)
  :safe 'listp)

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
;; FIXME: Using the current approach, major modes are supposed to set
;; this variable to a buffer-local value.  So we don't have access to
;; the "external roots" of language A from buffers of language B, which
;; seems desirable in multi-language projects, at least for some
;; potential uses, like "jump to a file in project or external dirs".
;;
;; We could add a second argument to this function: a file extension,
;; or a language name.  Some projects will know the set of languages
;; used in them; for others, like VC-based projects, we'll need
;; auto-detection.  I see two options:
;;
;; - That could be implemented as a separate second hook, with a
;;   list of functions that return file extensions.
;;
;; - This variable will be turned into a hook with "append" semantics,
;;   and each function in it will perform auto-detection when passed
;;   nil instead of an actual file extension.  Then this hook will, in
;;   general, be modified globally, and not from major mode functions.
;;
;; The second option seems simpler, but the first one has the
;; advantage that the user could override the list of languages used
;; in a project via a directory-local variable, thus skipping
;; languages they're not working on personally (in a big project), or
;; working around problems in language detection (the detection logic
;; might be imperfect for the project in question, or it might work
;; too slowly for the user's taste).
(defvar project-vc-external-roots-function (lambda () tags-table-list)
  "Function that returns a list of external roots.

It should return a list of directory roots that contain source
files related to the current buffer.

The directory names should be absolute.  Used in the VC project
backend implementation of `project-external-roots'.")

Dmitry Gutov's avatar
Dmitry Gutov committed
257 258 259 260 261 262
(defun project-try-vc (dir)
  (let* ((backend (ignore-errors (vc-responsible-backend dir)))
         (root (and backend (ignore-errors
                              (vc-call-backend backend 'root dir)))))
    (and root (cons 'vc root))))

263 264
(cl-defmethod project-roots ((project (head vc)))
  (list (cdr project)))
Dmitry Gutov's avatar
Dmitry Gutov committed
265

266
(cl-defmethod project-external-roots ((project (head vc)))
267 268
  (project-subtract-directories
   (project-combine-directories
269 270 271
    (mapcar
     #'file-name-as-directory
     (funcall project-vc-external-roots-function)))
272
   (project-roots project)))
273

274
(cl-defmethod project-ignores ((project (head vc)) dir)
275
  (let* ((root (cdr project))
276
          backend)
277
    (append
278 279 280 281 282 283 284
     (when (file-equal-p dir root)
       (setq backend (vc-responsible-backend root))
       (mapcar
        (lambda (entry)
          (if (string-match "\\`/" entry)
              (replace-match "./" t t entry)
            entry))
285 286
        (vc-call-backend backend 'ignore-completion-table root)))
     (project--value-in-dir 'project-vc-ignores root)
287 288 289 290
     (mapcar
      (lambda (dir)
        (concat dir "/"))
      vc-directory-exclusion-list))))
Dmitry Gutov's avatar
Dmitry Gutov committed
291

292 293 294 295 296
(defun project-combine-directories (&rest lists-of-dirs)
  "Return a sorted and culled list of directory names.
Appends the elements of LISTS-OF-DIRS together, removes
non-existing directories, as well as directories a parent of
whose is already in the list."
Dmitry Gutov's avatar
Dmitry Gutov committed
297 298 299 300
  (let* ((dirs (sort
                (mapcar
                 (lambda (dir)
                   (file-name-as-directory (expand-file-name dir)))
301
                 (apply #'append lists-of-dirs))
Dmitry Gutov's avatar
Dmitry Gutov committed
302 303 304 305 306 307 308 309 310
                #'string<))
         (ref dirs))
    ;; Delete subdirectories from the list.
    (while (cdr ref)
      (if (string-prefix-p (car ref) (cadr ref))
          (setcdr ref (cddr ref))
        (setq ref (cdr ref))))
    (cl-delete-if-not #'file-exists-p dirs)))

311 312 313
(defun project-subtract-directories (files dirs)
  "Return a list of elements from FILES that are outside of DIRS.
DIRS must contain directory names."
314 315
  ;; Sidestep the issue of expanded/abbreviated file names here.
  (cl-set-difference files dirs :test #'file-in-directory-p))
316

317 318 319
(defun project--value-in-dir (var dir)
  (with-temp-buffer
    (setq default-directory dir)
320 321
    (let ((enable-local-variables :all))
      (hack-dir-local-variables-non-file-buffer))
322 323
    (symbol-value var)))

324 325
(declare-function grep-read-files "grep")
(declare-function xref--show-xrefs "xref")
326
(declare-function xref-backend-identifier-at-point "xref")
327
(declare-function xref--find-ignores-arguments "xref")
328 329 330

;;;###autoload
(defun project-find-regexp (regexp)
331
  "Find all matches for REGEXP in the current project's roots.
332
With \\[universal-argument] prefix, you can specify the directory
333 334 335 336 337
to search in, and the file name pattern to search for.  The
pattern may use abbreviations defined in `grep-files-aliases',
e.g. entering `ch' is equivalent to `*.[ch]'.  As whitespace
triggers completion when entering a pattern, including it
requires quoting, e.g. `\\[quoted-insert]<space>'."
338
  (interactive (list (project--read-regexp)))
339
  (let* ((pr (project-current t))
340 341 342 343 344 345 346 347 348 349 350 351 352
         (files
          (if (not current-prefix-arg)
              (project-files pr (project-roots pr))
            (let ((dir (read-directory-name "Base directory: "
                                            nil default-directory t)))
              (project--files-in-directory dir
                                           (project--dir-ignores pr dir)
                                           (grep-read-files regexp))))))
    (project--find-regexp-in-files regexp files)))

(defun project--dir-ignores (project dir)
  (let* ((roots (project-roots project))
         (root (cl-find dir roots :test #'file-in-directory-p)))
353 354
    (if (not root)
        (project-ignores nil nil)       ;The defaults.
355 356 357 358 359 360
      (let ((ignores (project-ignores project root)))
        (if (file-equal-p root dir)
            ignores
          ;; FIXME: Update the "rooted" ignores to relate to DIR instead.
          (cl-delete-if (lambda (str) (string-prefix-p "./" str))
                        ignores))))))
361 362

;;;###autoload
363 364
(defun project-or-external-find-regexp (regexp)
  "Find all matches for REGEXP in the project roots or external roots.
365 366 367
With \\[universal-argument] prefix, you can specify the file name
pattern to search for."
  (interactive (list (project--read-regexp)))
368
  (let* ((pr (project-current t))
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
         (files
          (project-files pr (append
                             (project-roots pr)
                             (project-external-roots pr)))))
    (project--find-regexp-in-files regexp files)))

(defun project--find-regexp-in-files (regexp files)
  (pcase-let*
      ((output (get-buffer-create " *project grep output*"))
       (`(,grep-re ,file-group ,line-group . ,_) (car grep-regexp-alist))
       (status nil)
       (hits nil)
       (xrefs nil)
       (command (format "xargs -0 grep %s -nHe %s"
                        (if (and case-fold-search
                                 (isearch-no-upper-case-p regexp t))
                            "-i"
                          "")
                        (shell-quote-argument (xref--regexp-to-extended regexp)))))
    (with-current-buffer output
      (erase-buffer)
      (with-temp-buffer
        (insert (mapconcat #'identity files "\0"))
        (setq status
              (project--process-file-region (point-min)
                                            (point-max)
                                            shell-file-name
                                            output
                                            nil
                                            shell-command-switch
                                            command)))
      (goto-char (point-min))
      (when (and (/= (point-min) (point-max))
                 (not (looking-at grep-re))
                 ;; TODO: Show these matches as well somehow?
                 (not (looking-at "Binary file .* matches")))
        (user-error "Search failed with status %d: %s" status
                    (buffer-substring (point-min) (line-end-position))))
      (while (re-search-forward grep-re nil t)
        (push (list (string-to-number (match-string line-group))
                    (match-string file-group)
                    (buffer-substring-no-properties (point) (line-end-position)))
              hits)))
    (setq xrefs (xref--convert-hits (nreverse hits) regexp))
    (unless xrefs
      (user-error "No matches for: %s" regexp))
    (xref--show-xrefs xrefs nil)))

(defun project--process-file-region (start end program
                                     &optional buffer display
                                     &rest args)
  ;; FIXME: This branching shouldn't be necessary, but
  ;; call-process-region *is* measurably faster, even for a program
  ;; doing some actual work (for a period of time). Even though
  ;; call-process-region also creates a temp file internally
  ;; (http://lists.gnu.org/archive/html/emacs-devel/2019-01/msg00211.html).
  (if (not (file-remote-p default-directory))
      (apply #'call-process-region
             start end program nil buffer display args)
    (let ((infile (make-temp-file "ppfr")))
      (unwind-protect
          (progn
            (write-region start end infile nil 'silent)
            (apply #'process-file program infile buffer display args))
        (delete-file infile)))))
434 435

(defun project--read-regexp ()
436 437
  (let ((id (xref-backend-identifier-at-point (xref-find-backend))))
    (read-regexp "Find regexp" (and id (regexp-quote id)))))
438

439
;;;###autoload
440
(defun project-find-file ()
Stephen Leake's avatar
Stephen Leake committed
441 442 443
  "Visit a file (with completion) in the current project's roots.
The completion default is the filename at point, if one is
recognized."
444 445 446
  (interactive)
  (let* ((pr (project-current t))
         (dirs (project-roots pr)))
Stephen Leake's avatar
Stephen Leake committed
447
    (project-find-file-in (thing-at-point 'filename) dirs pr)))
448

449
;;;###autoload
450
(defun project-or-external-find-file ()
Stephen Leake's avatar
Stephen Leake committed
451 452 453
  "Visit a file (with completion) in the current project's roots or external roots.
The completion default is the filename at point, if one is
recognized."
454 455 456 457 458
  (interactive)
  (let* ((pr (project-current t))
         (dirs (append
                (project-roots pr)
                (project-external-roots pr))))
Stephen Leake's avatar
Stephen Leake committed
459
    (project-find-file-in (thing-at-point 'filename) dirs pr)))
460

Stephen Leake's avatar
Stephen Leake committed
461
(defun project-find-file-in (filename dirs project)
462 463 464 465 466 467 468 469 470 471 472 473
  "Complete FILENAME in DIRS in PROJECT and visit the result."
  (let* ((table (project-file-completion-table project dirs))
         (file (project--completing-read-strict
                "Find file" table nil nil
                filename)))
    (if (string= file "")
        (user-error "You didn't specify the file")
      (find-file file))))

(defun project--completing-read-strict (prompt
                                        collection &optional predicate
                                        hist default inherit-input-method)
Dmitry Gutov's avatar
Dmitry Gutov committed
474 475 476 477
  ;; Tried both expanding the default before showing the prompt, and
  ;; removing it when it has no matches.  Neither seems natural
  ;; enough.  Removal is confusing; early expansion makes the prompt
  ;; too long.
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
  (let* ((common-parent-directory
          (let ((common-prefix (try-completion "" collection)))
            (if (> (length common-prefix) 0)
                (file-name-directory common-prefix))))
         (cpd-length (length common-parent-directory))
         (prompt (if (zerop cpd-length)
                     prompt
                   (concat prompt (format " in %s" common-parent-directory))))
         ;; XXX: This requires collection to be "flat" as well.
         (substrings (mapcar (lambda (s) (substring s cpd-length))
                             (all-completions "" collection)))
         (new-collection
          (lambda (string pred action)
            (cond
             ((eq action 'metadata)
              (if (functionp collection) (funcall collection nil nil 'metadata)))
             (t
	      (complete-with-action action substrings string pred)))))
496
         (new-prompt (if default
497 498 499
                         (format "%s (default %s): " prompt default)
                       (format "%s: " prompt)))
         (res (completing-read new-prompt
500
                               new-collection predicate t
501 502
                               nil ;; initial-input
                               hist default inherit-input-method)))
503 504 505 506 507 508 509
    (when (and (equal res default)
               (not (test-completion res collection predicate)))
      (setq res
            (completing-read (format "%s: " prompt)
                             new-collection predicate t res hist nil
                             inherit-input-method)))
    (concat common-parent-directory res)))
510

511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535
(declare-function multifile-continue "multifile" ())

;;;###autoload
(defun project-search (regexp)
  "Search for REGEXP in all the files of the project.
Stops when a match is found.
To continue searching for next match, use command \\[multifile-continue]."
  (interactive "sSearch (regexp): ")
  (multifile-initialize-search
   regexp (project-files (project-current t)) 'default)
  (multifile-continue))

;;;###autoload
(defun project-query-replace (from to)
  "Search for REGEXP in all the files of the project.
Stops when a match is found.
To continue searching for next match, use command \\[multifile-continue]."
  (interactive
   (pcase-let ((`(,from ,to)
                (query-replace-read-args "Query replace (regexp)" t t)))
     (list from to)))
  (multifile-initialize-replace
   from to (project-files (project-current t)) 'default)
  (multifile-continue))

Dmitry Gutov's avatar
Dmitry Gutov committed
536 537
(provide 'project)
;;; project.el ends here