package.el 153 KB
Newer Older
1
;;; package.el --- Simple package system for Emacs  -*- lexical-binding:t -*-
2

3
;; Copyright (C) 2007-2019 Free Software Foundation, Inc.
4 5

;; Author: Tom Tromey <tromey@redhat.com>
6
;;         Daniel Hackney <dan@haxney.org>
7
;; Created: 10 Mar 2007
8
;; Version: 1.1.0
9
;; Keywords: tools
10
;; Package-Requires: ((tabulated-list "1.0"))
11 12 13

;; This file is part of GNU Emacs.

14
;; GNU Emacs is free software: you can redistribute it and/or modify
15
;; it under the terms of the GNU General Public License as published by
16 17
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
18 19 20 21 22 23 24

;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
25
;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46

;;; Commentary:

;; The idea behind package.el is to be able to download packages and
;; install them.  Packages are versioned and have versioned
;; dependencies.  Furthermore, this supports built-in packages which
;; may or may not be newer than user-specified packages.  This makes
;; it possible to upgrade Emacs and automatically disable packages
;; which have moved from external to core.  (Note though that we don't
;; currently register any of these, so this feature does not actually
;; work.)

;; A package is described by its name and version.  The distribution
;; format is either  a tar file or a single .el file.

;; A tar file should be named "NAME-VERSION.tar".  The tar file must
;; unpack into a directory named after the package and version:
;; "NAME-VERSION".  It must contain a file named "PACKAGE-pkg.el"
;; which consists of a call to define-package.  It may also contain a
;; "dir" file and the info files it references.

47
;; A .el file is named "NAME-VERSION.el" in the remote archive, but is
48 49
;; installed as simply "NAME.el" in a directory named "NAME-VERSION".

50 51 52 53
;; The downloader downloads all dependent packages.  By default,
;; packages come from the official GNU sources, but others may be
;; added by customizing the `package-archives' alist.  Packages get
;; byte-compiled at install time.
54 55 56 57 58 59 60 61 62 63

;; At activation time we will set up the load-path and the info path,
;; and we will load the package's autoloads.  If a package's
;; dependencies are not available, we will not activate that package.

;; Conceptually a package has multiple state transitions:
;;
;; * Download.  Fetching the package from ELPA.
;; * Install.  Untar the package, or write the .el file, into
;;   ~/.emacs.d/elpa/ directory.
64
;; * Autoload generation.
65 66 67 68 69 70 71 72
;; * Byte compile.  Currently this phase is done during install,
;;   but we may change this.
;; * Activate.  Evaluate the autoloads for the package to make it
;;   available to the user.
;; * Load.  Actually load the package and run some code from it.

;; Other external functions you may want to use:
;;
73
;; M-x list-packages
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
;;    Enters a mode similar to buffer-menu which lets you manage
;;    packages.  You can choose packages for install (mark with "i",
;;    then "x" to execute) or deletion (not implemented yet), and you
;;    can see what packages are available.  This will automatically
;;    fetch the latest list of packages from ELPA.
;;
;; M-x package-install-from-buffer
;;    Install a package consisting of a single .el file that appears
;;    in the current buffer.  This only works for packages which
;;    define a Version header properly; package.el also supports the
;;    extension headers Package-Version (in case Version is an RCS id
;;    or similar), and Package-Requires (if the package requires other
;;    packages).
;;
;; M-x package-install-file
;;    Install a package from the indicated file.  The package can be
;;    either a tar file or a .el file.  A tar file must contain an
;;    appropriately-named "-pkg.el" file; a .el file must be properly
;;    formatted as with package-install-from-buffer.

;;; Thanks:
;;; (sorted by sort-lines):

;; Jim Blandy <jimb@red-bean.com>
;; Karl Fogel <kfogel@red-bean.com>
;; Kevin Ryde <user42@zip.com.au>
;; Lawrence Mitchell
;; Michael Olson <mwolson@member.fsf.org>
;; Sebastian Tennant <sebyte@smolny.plus.com>
;; Stefan Monnier <monnier@iro.umontreal.ca>
viniciusjl's avatar
viniciusjl committed
104
;; Vinicius Jose Latorre <viniciusjl.gnu@gmail.com>
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
;; Phil Hagelberg <phil@hagelb.org>

;;; ToDo:

;; - putting info dirs at the start of the info path means
;;   users see a weird ordering of categories.  OTOH we want to
;;   override later entries.  maybe emacs needs to enforce
;;   the standard layout?
;; - put bytecode in a separate directory tree
;; - perhaps give users a way to recompile their bytecode
;;   or do it automatically when emacs changes
;; - give users a way to know whether a package is installed ok
;; - give users a way to view a package's documentation when it
;;   only appears in the .el
;; - use/extend checkdoc so people can tell if their package will work
;; - "installed" instead of a blank in the status column
;; - tramp needs its files to be compiled in a certain order.
;;   how to handle this?  fix tramp?
;; - maybe we need separate .elc directories for various emacs versions
;;   and also emacs-vs-xemacs.  That way conditional compilation can
;;   work.  But would this break anything?
;; - William Xu suggests being able to open a package file without
;;   installing it
;; - Interface with desktop.el so that restarting after an install
;;   works properly
;; - Use hierarchical layout.  PKG/etc PKG/lisp PKG/info
;;   ... except maybe lisp?
;; - It may be nice to have a macro that expands to the package's
;;   private data dir, aka ".../etc".  Or, maybe data-directory
;;   needs to be a list (though this would be less nice)
;;   a few packages want this, eg sokoban
136 137 138
;; - Allow multiple versions on the server, so that if a user doesn't
;;   meet the requirements for the most recent version they can still
;;   install an older one.
139 140 141 142 143 144 145
;; - Allow optional package dependencies
;;   then if we require 'bbdb', bbdb-specific lisp in lisp/bbdb
;;   and just don't compile to add to load path ...?
;; - Our treatment of the info path is somewhat bogus

;;; Code:

146
(require 'cl-lib)
147
(eval-when-compile (require 'subr-x))
148
(eval-when-compile (require 'epg))      ;For setf accessors.
149
(require 'seq)
150

151
(require 'tabulated-list)
152
(require 'macroexp)
153
(require 'url-handlers)
154

155 156 157 158 159
(defgroup package nil
  "Manager for Emacs Lisp packages."
  :group 'applications
  :version "24.1")

160 161

;;; Customization options
162 163
;;;###autoload
(defcustom package-enable-at-startup t
164 165 166 167 168 169 170
  "Whether to make installed packages available when Emacs starts.
If non-nil, packages are made available before reading the init
file (but after reading the early init file).  This means that if
you wish to set this variable, you must do so in the early init
file.  Regardless of the value of this variable, packages are not
made available if `user-init-file' is nil (e.g. Emacs was started
with \"-q\").
171 172

Even if the value is nil, you can type \\[package-initialize] to
173 174
make installed packages available at any time, or you can
call (package-initialize) in your init-file."
175 176 177 178
  :type 'boolean
  :version "24.1")

(defcustom package-load-list '(all)
179
  "List of packages for `package-initialize' to make available.
180
Each element in this list should be a list (NAME VERSION), or the
181 182 183
symbol `all'.  The symbol `all' says to make available the latest
installed versions of all packages not specified by other
elements.
184 185 186

For an element (NAME VERSION), NAME is a package name (a symbol).
VERSION should be t, a string, or nil.
187 188
If VERSION is t, the most recent version is made available.
If VERSION is a string, only that version is ever made available.
189 190
 Any other version, even if newer, is silently ignored.
 Hence, the package is \"held\" at that version.
191
If VERSION is nil, the package is not made available (it is \"disabled\")."
192 193 194 195 196 197 198
  :type '(repeat (choice (const all)
                         (list :tag "Specific package"
                               (symbol :tag "Package name")
                               (choice :tag "Version"
                                (const :tag "disable" nil)
                                (const :tag "most recent" t)
                                (string :tag "specific version")))))
199
  :risky t
200 201
  :version "24.1")

202 203 204
(defcustom package-archives `(("gnu" .
                               ,(format "http%s://elpa.gnu.org/packages/"
                                        (if (gnutls-available-p) "s" ""))))
205 206
  "An alist of archives from which to fetch.
The default value points to the GNU Emacs package repository.
207 208 209 210

Each element has the form (ID . LOCATION).
 ID is an archive name, as a string.
 LOCATION specifies the base location for the archive.
211
  If it starts with \"http(s):\", it is treated as an HTTP(S) URL;
212
  otherwise it should be an absolute directory name.
213 214 215 216
  (Other types of URL are currently not supported.)

Only add locations that you trust, since fetching and installing
a package can run arbitrary code."
217
  :type '(alist :key-type (string :tag "Archive name")
218
                :value-type (string :tag "URL or directory name"))
219
  :risky t
220
  :version "26.1")                      ; gnutls test
221

222 223 224 225 226 227 228 229 230 231 232 233
(defcustom package-menu-hide-low-priority 'archive
  "If non-nil, hide low priority packages from the packages menu.
A package is considered low priority if there's another version
of it available such that:
    (a) the archive of the other package is higher priority than
    this one, as per `package-archive-priorities';
  or
    (b) they both have the same archive priority but the other
    package has a higher version number.

This variable has three possible values:
    nil: no packages are hidden;
234
    `archive': only criterion (a) is used;
235 236
    t: both criteria are used.

237 238
This variable has no effect if `package-menu--hide-packages' is
nil, so it can be toggled with \\<package-menu-mode-map> \\[package-menu-toggle-hiding]."
239 240 241 242 243 244
  :type '(choice (const :tag "Don't hide anything" nil)
                 (const :tag "Hide per package-archive-priorities"
                        archive)
                 (const :tag "Hide per archive and version number" t))
  :version "25.1")

245 246 247 248 249 250 251
(defcustom package-archive-priorities nil
  "An alist of priorities for packages.

Each element has the form (ARCHIVE-ID . PRIORITY).

When installing packages, the package with the highest version
number from the archive with the highest priority is
252
selected.  When higher versions are available from archives with
253 254
lower priorities, the user has to select those manually.

255 256 257
Archives not in this list have the priority 0.

See also `package-menu-hide-low-priority'."
258 259
  :type '(alist :key-type (string :tag "Archive name")
                :value-type (integer :tag "Priority (default is 0)"))
260 261 262
  :risky t
  :version "25.1")

263
(defcustom package-pinned-packages nil
264 265 266 267 268 269 270 271 272 273 274 275 276
  "An alist of packages that are pinned to specific archives.
This can be useful if you have multiple package archives enabled,
and want to control which archive a given package gets installed from.

Each element of the alist has the form (PACKAGE . ARCHIVE), where:
 PACKAGE is a symbol representing a package
 ARCHIVE is a string representing an archive (it should be the car of
an element in `package-archives', e.g. \"gnu\").

Adding an entry to this variable means that only ARCHIVE will be
considered as a source for PACKAGE.  If other archives provide PACKAGE,
they are ignored (for this package).  If ARCHIVE does not contain PACKAGE,
the package will be unavailable."
277 278
  :type '(alist :key-type (symbol :tag "Package")
                :value-type (string :tag "Archive name"))
279 280 281 282
  ;; I don't really see why this is risky...
  ;; I suppose it could prevent you receiving updates for a package,
  ;; via an entry (PACKAGE . NON-EXISTING).  Which could be an issue
  ;; if PACKAGE has a known vulnerability that is fixed in newer versions.
283 284 285
  :risky t
  :version "24.4")

286 287 288 289 290 291
(defcustom package-user-dir (locate-user-emacs-file "elpa")
  "Directory containing the user's Emacs Lisp packages.
The directory name should be absolute.
Apart from this directory, Emacs also looks for system-wide
packages in `package-directory-list'."
  :type 'directory
292
  :risky t
293 294 295 296 297 298
  :version "24.1")

(defcustom package-directory-list
  ;; Defaults are subdirs named "elpa" in the site-lisp dirs.
  (let (result)
    (dolist (f load-path)
299
      (and (stringp f)
300 301
           (equal (file-name-nondirectory f) "site-lisp")
           (push (expand-file-name "elpa" f) result)))
302 303 304 305 306 307 308
    (nreverse result))
  "List of additional directories containing Emacs Lisp packages.
Each directory name should be absolute.

These directories contain packages intended for system-wide; in
contrast, `package-user-dir' contains packages for personal use."
  :type '(repeat directory)
309
  :risky t
310 311
  :version "24.1")

312
(declare-function epg-find-configuration "epg-config"
313
                  (protocol &optional no-cache program-alist))
Stefan Monnier's avatar
Stefan Monnier committed
314

315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
(defcustom package-gnupghome-dir (expand-file-name "gnupg" package-user-dir)
  "Directory containing GnuPG keyring or nil.
This variable specifies the GnuPG home directory used by package.
That directory is passed via the option \"--homedir\" to GnuPG.
If nil, do not use the option \"--homedir\", but stick with GnuPG's
default directory."
  :type `(choice
          (const
           :tag "Default Emacs package management GnuPG home directory"
           ,(expand-file-name "gnupg" package-user-dir))
          (const
           :tag "Default GnuPG directory (GnuPG option --homedir not used)"
           nil)
          (directory :tag "A specific GnuPG --homedir"))
  :risky t
  :version "26.1")

332
(defcustom package-check-signature
333 334
  (if (and (require 'epg-config)
           (epg-find-configuration 'OpenPGP))
335
      'allow-unsigned)
336
  "Non-nil means to check package signatures when installing.
337 338 339 340 341 342 343
More specifically the value can be:
- nil: package signatures are ignored.
- `allow-unsigned': install a package even if it is unsigned,
  but if it is signed and we have the key for it, verify the signature.
- t: accept a package only if it comes with at least one verified signature.
- `all': same as t, except when the package has several signatures,
  in which case we verify all the signatures.
344 345 346

This also applies to the \"archive-contents\" file that lists the
contents of the archive."
347
  :type '(choice (const nil :tag "Never")
348
                 (const allow-unsigned :tag "Allow unsigned")
349 350
                 (const t :tag "Check always")
                 (const all :tag "Check all signatures"))
351
  :risky t
352
  :version "27.1")
353 354

(defcustom package-unsigned-archives nil
355
  "List of archives where we do not check for package signatures."
356 357
  :type '(repeat (string :tag "Archive name"))
  :risky t
358
  :version "24.4")
359

360
(defcustom package-selected-packages nil
Paul Eggert's avatar
Paul Eggert committed
361 362 363 364
  "Store here packages installed explicitly by user.
This variable is fed automatically by Emacs when installing a new package.
This variable is used by `package-autoremove' to decide
which packages are no longer needed.
365
You can use it to (re)install packages on other machines
366
by running `package-install-selected-packages'.
367 368 369 370

To check if a package is contained in this list here, use
`package--user-selected-p', as it may populate the variable with
a sane initial value."
371
  :version "25.1"
372
  :type '(repeat symbol))
373

374 375
(defcustom package-menu-async t
  "If non-nil, package-menu will use async operations when possible.
376 377 378
Currently, only the refreshing of archive contents supports
asynchronous operations.  Package transactions are still done
synchronously."
379 380 381
  :type 'boolean
  :version "25.1")

382 383 384 385 386 387 388 389

;;; `package-desc' object definition
;; This is the struct used internally to represent packages.
;; Functions that deal with packages should generally take this object
;; as an argument.  In some situations (e.g. commands that query the
;; user) it makes sense to take the package name as a symbol instead,
;; but keep in mind there could be multiple `package-desc's with the
;; same name.
390 391 392 393 394 395 396 397 398 399 400
(defvar package--default-summary "No description available.")

(cl-defstruct (package-desc
               ;; Rename the default constructor from `make-package-desc'.
               (:constructor package-desc-create)
               ;; Has the same interface as the old `define-package',
               ;; which is still used in the "foo-pkg.el" files. Extra
               ;; options can be supported by adding additional keys.
               (:constructor
                package-desc-from-define
                (name-string version-string &optional summary requirements
401
                 &rest rest-plist
402 403 404 405 406 407 408 409
                 &aux
                 (name (intern name-string))
                 (version (version-to-list version-string))
                 (reqs (mapcar #'(lambda (elt)
                                   (list (car elt)
                                         (version-to-list (cadr elt))))
                               (if (eq 'quote (car requirements))
                                   (nth 1 requirements)
410 411 412 413 414
                                 requirements)))
                 (kind (plist-get rest-plist :kind))
                 (archive (plist-get rest-plist :archive))
                 (extras (let (alist)
                           (while rest-plist
415 416 417
                             (unless (memq (car rest-plist) '(:kind :archive))
                               (let ((value (cadr rest-plist)))
                                 (when value
418 419
                                   (push (cons (car rest-plist)
                                               (if (eq (car-safe value) 'quote)
Dmitry Gutov's avatar
Dmitry Gutov committed
420
                                                   (cadr value)
421
                                                 value))
422
                                         alist))))
423 424
                             (setq rest-plist (cddr rest-plist)))
                           alist)))))
425 426 427
  "Structure containing information about an individual package.
Slots:

428
`name'	Name of the package, as a symbol.
429 430 431 432

`version' Version of the package, as a version list.

`summary' Short description of the package, typically taken from
433
        the first line of the file.
434

435
`reqs'	Requirements of the package. A list of (PACKAGE
436 437
        VERSION-LIST) naming the dependent package and the minimum
        required version.
438

439
`kind'	The distribution format of the package. Currently, it is
440
        either `single' or `tar'.
441 442

`archive' The name of the archive (as a string) whence this
443
        package came.
444

445
`dir'	The directory where the package is installed (if installed),
446
        `builtin' if it is built-in, or nil otherwise.
447

448 449 450
`extras' Optional alist of additional keyword-value pairs.

`signed' Flag to indicate that the package is signed by provider."
451 452 453 454 455
  name
  version
  (summary package--default-summary)
  reqs
  kind
456
  archive
457
  dir
458 459
  extras
  signed)
460

461 462 463 464 465 466
(defun package--from-builtin (bi-desc)
  (package-desc-create :name (pop bi-desc)
                       :version (package--bi-desc-version bi-desc)
                       :summary (package--bi-desc-summary bi-desc)
                       :dir 'builtin))

467
;; Pseudo fields.
468 469 470 471 472 473 474 475 476 477 478 479 480 481
(defun package-version-join (vlist)
  "Return the version string corresponding to the list VLIST.
This is, approximately, the inverse of `version-to-list'.
\(Actually, it returns only one of the possible inverses, since
`version-to-list' is a many-to-one operation.)"
  (if (null vlist)
      ""
    (let ((str-list (list "." (int-to-string (car vlist)))))
      (dolist (num (cdr vlist))
        (cond
         ((>= num 0)
          (push (int-to-string num) str-list)
          (push "." str-list))
         ((< num -4)
482
          (error "Invalid version list `%s'" vlist))
483 484 485 486 487
         (t
          ;; pre, or beta, or alpha
          (cond ((equal "." (car str-list))
                 (pop str-list))
                ((not (string-match "[0-9]+" (car str-list)))
488
                 (error "Invalid version list `%s'" vlist)))
489 490 491 492 493 494 495
          (push (cond ((= num -1) "pre")
                      ((= num -2) "beta")
                      ((= num -3) "alpha")
                      ((= num -4) "snapshot"))
                str-list))))
      (if (equal "." (car str-list))
          (pop str-list))
496
      (apply #'concat (nreverse str-list)))))
497

498
(defun package-desc-full-name (pkg-desc)
499 500 501
  (format "%s-%s"
          (package-desc-name pkg-desc)
          (package-version-join (package-desc-version pkg-desc))))
502

503 504
(defun package-desc-suffix (pkg-desc)
  (pcase (package-desc-kind pkg-desc)
505 506 507
    ('single ".el")
    ('tar ".tar")
    ('dir "")
508 509
    (kind (error "Unknown package kind: %s" kind))))

510 511
(defun package-desc--keywords (pkg-desc)
  (let ((keywords (cdr (assoc :keywords (package-desc-extras pkg-desc)))))
512
    (if (eq (car-safe keywords) 'quote)
513
        (nth 1 keywords)
514 515
      keywords)))

516 517 518 519
(defun package-desc-priority (p)
  "Return the priority of the archive of package-desc object P."
  (package-archive-priority (package-desc-archive p)))

520 521 522 523 524 525 526 527
;; Package descriptor format used in finder-inf.el and package--builtins.
(cl-defstruct (package--bi-desc
               (:constructor package-make-builtin (version summary))
               (:type vector))
  version
  reqs
  summary)

528 529 530 531 532 533 534

;;; Installed packages
;; The following variables store information about packages present in
;; the system.  The most important of these is `package-alist'.  The
;; command `package-initialize' is also closely related to this
;; section, but it is left for a later section because it also affects
;; other stuff.
535 536
(defvar package--builtins nil
  "Alist of built-in packages.
537 538 539 540
The actual value is initialized by loading the library
`finder-inf'; this is not done until it is needed, e.g. by the
function `package-built-in-p'.

541 542
Each element has the form (PKG . PACKAGE-BI-DESC), where PKG is a package
name (a symbol) and DESC is a `package--bi-desc' structure.")
543
(put 'package--builtins 'risky-local-variable t)
544

545
(defvar package-alist nil
546
  "Alist of all packages available for activation.
547 548 549
Each element has the form (PKG . DESCS), where PKG is a package
name (a symbol) and DESCS is a non-empty list of `package-desc' structure,
sorted by decreasing versions.
550 551 552 553

This variable is set automatically by `package-load-descriptor',
called via `package-initialize'.  To change which packages are
loaded and/or activated, customize `package-load-list'.")
554
(put 'package-alist 'risky-local-variable t)
555

556
(defvar package-activated-list nil
557
  ;; FIXME: This should implicitly include all builtin packages.
558
  "List of the names of currently activated packages.")
559
(put 'package-activated-list 'risky-local-variable t)
560

561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
;;;; Populating `package-alist'.
;; The following functions are called on each installed package by
;; `package-load-all-descriptors', which ultimately populates the
;; `package-alist' variable.
(defun package-process-define-package (exp)
  (when (eq (car-safe exp) 'define-package)
    (let* ((new-pkg-desc (apply #'package-desc-from-define (cdr exp)))
           (name (package-desc-name new-pkg-desc))
           (version (package-desc-version new-pkg-desc))
           (old-pkgs (assq name package-alist)))
      (if (null old-pkgs)
          ;; If there's no old package, just add this to `package-alist'.
          (push (list name new-pkg-desc) package-alist)
        ;; If there is, insert the new package at the right place in the list.
        (while
            (if (and (cdr old-pkgs)
                     (version-list-< version
                                     (package-desc-version (cadr old-pkgs))))
                (setq old-pkgs (cdr old-pkgs))
              (push new-pkg-desc (cdr old-pkgs))
              nil)))
      new-pkg-desc)))
583

584 585 586
(defun package-load-descriptor (pkg-dir)
  "Load the description file in directory PKG-DIR."
  (let ((pkg-file (expand-file-name (package--description-file pkg-dir)
587
                                    pkg-dir))
588
        (signed-file (concat pkg-dir ".signed")))
589 590 591 592
    (when (file-exists-p pkg-file)
      (with-temp-buffer
        (insert-file-contents pkg-file)
        (goto-char (point-min))
593 594 595
        (let ((pkg-desc (or (package-process-define-package
                             (read (current-buffer)))
                            (error "Can't find define-package in %s" pkg-file))))
596
          (setf (package-desc-dir pkg-desc) pkg-dir)
597 598
          (if (file-exists-p signed-file)
              (setf (package-desc-signed pkg-desc) t))
599
          pkg-desc)))))
600 601 602 603 604 605 606 607 608

(defun package-load-all-descriptors ()
  "Load descriptors for installed Emacs Lisp packages.
This looks for package subdirectories in `package-user-dir' and
`package-directory-list'.  The variable `package-load-list'
controls which package subdirectories may be loaded.

In each valid package subdirectory, this function loads the
description file containing a call to `define-package', which
609
updates `package-alist'."
610 611 612
  (dolist (dir (cons package-user-dir package-directory-list))
    (when (file-directory-p dir)
      (dolist (subdir (directory-files dir))
613 614 615 616
        (unless (equal subdir "..")
          (let ((pkg-dir (expand-file-name subdir dir)))
            (when (file-directory-p pkg-dir)
              (package-load-descriptor pkg-dir))))))))
617

618 619 620 621 622 623
(defun package--alist ()
  "Return `package-alist', after computing it if needed."
  (or package-alist
      (progn (package-load-all-descriptors)
             package-alist)))

624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
(defun define-package (_name-string _version-string
                                    &optional _docstring _requirements
                                    &rest _extra-properties)
  "Define a new package.
NAME-STRING is the name of the package, as a string.
VERSION-STRING is the version of the package, as a string.
DOCSTRING is a short description of the package, a string.
REQUIREMENTS is a list of dependencies on other packages.
 Each requirement is of the form (OTHER-PACKAGE OTHER-VERSION),
 where OTHER-VERSION is a string.

EXTRA-PROPERTIES is currently unused."
  ;; FIXME: Placeholder!  Should we keep it?
  (error "Don't call me!"))


;;; Package activation
;; Section for functions used by `package-activate', which see.
642 643 644 645 646 647 648 649 650 651 652 653 654
(defun package-disabled-p (pkg-name version)
  "Return whether PKG-NAME at VERSION can be activated.
The decision is made according to `package-load-list'.
Return nil if the package can be activated.
Return t if the package is completely disabled.
Return the max version (as a string) if the package is held at a lower version."
  (let ((force (assq pkg-name package-load-list)))
    (cond ((null force) (not (memq 'all package-load-list)))
          ((null (setq force (cadr force))) t) ; disabled
          ((eq force t) nil)
          ((stringp force)              ; held
           (unless (version-list-= version (version-to-list force))
             force))
655
          (t (error "Invalid element in `package-load-list'")))))
656

657
(defun package-built-in-p (package &optional min-version)
658
  "Return non-nil if PACKAGE is built-in to Emacs.
659 660 661 662 663 664 665 666 667 668 669 670
Optional arg MIN-VERSION, if non-nil, should be a version list
specifying the minimum acceptable version."
  (if (package-desc-p package) ;; was built-in and then was converted
      (eq 'builtin (package-desc-dir package))
    (let ((bi (assq package package--builtin-versions)))
      (cond
       (bi (version-list-<= min-version (cdr bi)))
       ((remove 0 min-version) nil)
       (t
        (require 'finder-inf nil t) ; For `package--builtins'.
        (assq package package--builtins))))))

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
(defun package--autoloads-file-name (pkg-desc)
  "Return the absolute name of the autoloads file, sans extension.
PKG-DESC is a `package-desc' object."
  (expand-file-name
   (format "%s-autoloads" (package-desc-name pkg-desc))
   (package-desc-dir pkg-desc)))

(defun package--activate-autoloads-and-load-path (pkg-desc)
  "Load the autoloads file and add package dir to `load-path'.
PKG-DESC is a `package-desc' object."
  (let* ((old-lp load-path)
         (pkg-dir (package-desc-dir pkg-desc))
         (pkg-dir-dir (file-name-as-directory pkg-dir)))
    (with-demoted-errors "Error loading autoloads: %s"
      (load (package--autoloads-file-name pkg-desc) nil t))
    (when (and (eq old-lp load-path)
               (not (or (member pkg-dir load-path)
                        (member pkg-dir-dir load-path))))
      ;; Old packages don't add themselves to the `load-path', so we have to
      ;; do it ourselves.
      (push pkg-dir load-path))))

693 694 695
(defvar Info-directory-list)
(declare-function info-initialize "info" ())

696 697 698
(defvar package--quickstart-pkgs t
  "If set to a list, we're computing the set of pkgs to activate.")

699 700 701 702 703
(defun package--load-files-for-activation (pkg-desc reload)
  "Load files for activating a package given by PKG-DESC.
Load the autoloads file, and ensure `load-path' is setup.  If
RELOAD is non-nil, also load all files in the package that
correspond to previously loaded files."
704 705 706
  (let* ((loaded-files-list
          (when reload
            (package--list-loaded-files (package-desc-dir pkg-desc)))))
707 708 709 710 711 712 713 714 715 716 717 718 719 720
    ;; Add to load path, add autoloads, and activate the package.
    (package--activate-autoloads-and-load-path pkg-desc)
    ;; Call `load' on all files in `package-desc-dir' already present in
    ;; `load-history'.  This is done so that macros in these files are updated
    ;; to their new definitions.  If another package is being installed which
    ;; depends on this new definition, not doing this update would cause
    ;; compilation errors and break the installation.
    (with-demoted-errors "Error in package--load-files-for-activation: %s"
      (mapc (lambda (feature) (load feature nil t))
            ;; Skip autoloads file since we already evaluated it above.
            (remove (file-truename (package--autoloads-file-name pkg-desc))
                    loaded-files-list)))))

(defun package-activate-1 (pkg-desc &optional reload deps)
721
  "Activate package given by PKG-DESC, even if it was already active.
722 723
If DEPS is non-nil, also activate its dependencies (unless they
are already activated).
724 725 726
If RELOAD is non-nil, also `load' any files inside the package which
correspond to previously loaded files (those returned by
`package--list-loaded-files')."
727
  (let* ((name (package-desc-name pkg-desc))
728
         (pkg-dir (package-desc-dir pkg-desc)))
729
    (unless pkg-dir
730
      (error "Internal error: unable to find directory for `%s'"
731
             (package-desc-full-name pkg-desc)))
732 733 734 735 736 737 738 739 740 741
    (catch 'exit
      ;; Activate its dependencies recursively.
      ;; FIXME: This doesn't check whether the activated version is the
      ;; required version.
      (when deps
        (dolist (req (package-desc-reqs pkg-desc))
          (unless (package-activate (car req))
            (message "Unable to activate package `%s'.\nRequired package `%s-%s' is unavailable"
                     name (car req) (package-version-join (cadr req)))
            (throw 'exit nil))))
742 743 744 745
      (if (listp package--quickstart-pkgs)
          ;; We're only collecting the set of packages to activate!
          (push pkg-desc package--quickstart-pkgs)
        (package--load-files-for-activation pkg-desc reload))
746 747 748 749 750 751 752 753 754
      ;; Add info node.
      (when (file-exists-p (expand-file-name "dir" pkg-dir))
        ;; FIXME: not the friendliest, but simple.
        (require 'info)
        (info-initialize)
        (push pkg-dir Info-directory-list))
      (push name package-activated-list)
      ;; Don't return nil.
      t)))
755

756
(declare-function find-library-name "find-func" (library))
757

758 759 760 761
(defun package--list-loaded-files (dir)
  "Recursively list all files in DIR which correspond to loaded features.
Returns the `file-name-sans-extension' of each file, relative to
DIR, sorted by most recently loaded last."
762 763 764
  (let* ((history (delq nil
                        (mapcar (lambda (x)
                                  (let ((f (car x)))
765 766
                                    (and (stringp f)
                                         (file-name-sans-extension f))))
767
                                load-history)))
768 769 770
         (dir (file-truename dir))
         ;; List all files that have already been loaded.
         (list-of-conflicts
771
          (delq
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792
           nil
           (mapcar
               (lambda (x) (let* ((file (file-relative-name x dir))
                             ;; Previously loaded file, if any.
                             (previous
                              (ignore-errors
                                (file-name-sans-extension
                                 (file-truename (find-library-name file)))))
                             (pos (when previous (member previous history))))
                        ;; Return (RELATIVE-FILENAME . HISTORY-POSITION)
                        (when pos
                          (cons (file-name-sans-extension file) (length pos)))))
             (directory-files-recursively dir "\\`[^\\.].*\\.el\\'")))))
    ;; Turn the list of (FILENAME . POS) back into a list of features.  Files in
    ;; subdirectories are returned relative to DIR (so not actually features).
    (let ((default-directory (file-name-as-directory dir)))
      (mapcar (lambda (x) (file-truename (car x)))
        (sort list-of-conflicts
              ;; Sort the files by ascending HISTORY-POSITION.
              (lambda (x y) (< (cdr x) (cdr y))))))))

793 794 795 796
;;;; `package-activate'
;; This function activates a newer version of a package if an older
;; one was already activated.  It also loads a features of this
;; package which were already loaded.
797
(defun package-activate (package &optional force)
798
  "Activate the package named PACKAGE.
799 800
If FORCE is true, (re-)activate it if it's already activated.
Newer versions are always activated, regardless of FORCE."
801
  (let ((pkg-descs (cdr (assq package package-alist))))
802
    ;; Check if PACKAGE is available in `package-alist'.
803 804 805 806 807 808 809
    (while
        (when pkg-descs
          (let ((available-version (package-desc-version (car pkg-descs))))
            (or (package-disabled-p package available-version)
                ;; Prefer a builtin package.
                (package-built-in-p package available-version))))
      (setq pkg-descs (cdr pkg-descs)))
810 811
    (cond
     ;; If no such package is found, maybe it's built-in.
812 813
     ((null pkg-descs)
      (package-built-in-p package))
814
     ;; If the package is already activated, just return t.
815
     ((and (memq package package-activated-list) (not force))
816 817
      t)
     ;; Otherwise, proceed with activation.
818
     (t (package-activate-1 (car pkg-descs) nil 'deps)))))
819

820 821 822 823 824 825

;;; Installation -- Local operations
;; This section contains a variety of features regarding installing a
;; package to/from disk.  This includes autoload generation,
;; unpacking, compiling, as well as defining a package from the
;; current buffer.
826

827
;;;; Unpacking
828 829
(defvar tar-parse-info)
(declare-function tar-untar-buffer "tar-mode" ())
Glenn Morris's avatar
Glenn Morris committed
830 831
(declare-function tar-header-name "tar-mode" (tar-header) t)
(declare-function tar-header-link-type "tar-mode" (tar-header) t)
832 833

(defun package-untar-buffer (dir)
834
  "Untar the current buffer.
835 836
This uses `tar-untar-buffer' from Tar mode.  All files should
untar into a directory named DIR; otherwise, signal an error."
837
  (require 'tar-mode)
838 839
  (tar-mode)
  ;; Make sure everything extracts into DIR.
840
  (let ((regexp (concat "\\`" (regexp-quote (expand-file-name dir)) "/"))
841
        (case-fold-search (file-name-case-insensitive-p dir)))
842
    (dolist (tar-data tar-parse-info)
843
      (let ((name (expand-file-name (tar-header-name tar-data))))
844 845 846 847 848 849
        (or (string-match regexp name)
            ;; Tarballs created by some utilities don't list
            ;; directories with a trailing slash (Bug#13136).
            (and (string-equal dir name)
                 (eq (tar-header-link-type tar-data) 5))
            (error "Package does not untar cleanly into directory %s/" dir)))))
850
  (tar-untar-buffer))
851

852
(defun package--alist-to-plist-args (alist)
853
  (mapcar #'macroexp-quote
854 855
          (apply #'nconc
                 (mapcar (lambda (pair) (list (car pair) (cdr pair))) alist))))
856 857 858 859
(defun package-unpack (pkg-desc)
  "Install the contents of the current buffer as a package."
  (let* ((name (package-desc-name pkg-desc))
         (dirname (package-desc-full-name pkg-desc))
860
         (pkg-dir (expand-file-name dirname package-user-dir)))
861
    (pcase (package-desc-kind pkg-desc)
862
      ('dir
863 864 865 866 867 868 869 870 871 872 873 874 875
       (make-directory pkg-dir t)
       (let ((file-list
              (directory-files
               default-directory 'full "\\`[^.].*\\.el\\'" 'nosort)))
         (dolist (source-file file-list)
           (let ((target-el-file
                  (expand-file-name (file-name-nondirectory source-file) pkg-dir)))
             (copy-file source-file target-el-file t)))
         ;; Now that the files have been installed, this package is
         ;; indistinguishable from a `tar' or a `single'. Let's make
         ;; things simple by ensuring we're one of them.
         (setf (package-desc-kind pkg-desc)
               (if (> (length file-list) 1) 'tar 'single))))
876
      ('tar
877 878 879 880
       (make-directory package-user-dir t)
       ;; FIXME: should we delete PKG-DIR if it exists?
       (let* ((default-directory (file-name-as-directory package-user-dir)))
         (package-untar-buffer dirname)))
881
      ('single
882 883 884 885 886 887 888
       (let ((el-file (expand-file-name (format "%s.el" name) pkg-dir)))
         (make-directory pkg-dir t)
         (package--write-file-no-coding el-file)))
      (kind (error "Unknown package kind: %S" kind)))
    (package--make-autoloads-and-stuff pkg-desc pkg-dir)
    ;; Update package-alist.
    (let ((new-desc (package-load-descriptor pkg-dir)))
889 890 891 892
      (unless (equal (package-desc-full-name new-desc)
                     (package-desc-full-name pkg-desc))
        (error "The retrieved package (`%s') doesn't match what the archive offered (`%s')"
               (package-desc-full-name new-desc) (package-desc-full-name pkg-desc)))
893 894 895
      ;; Activation has to be done before compilation, so that if we're
      ;; upgrading and macros have changed we load the new definitions
      ;; before compiling.
896 897 898 899 900 901 902 903
      (when (package-activate-1 new-desc :reload :deps)
        ;; FIXME: Compilation should be done as a separate, optional, step.
        ;; E.g. for multi-package installs, we should first install all packages
        ;; and then compile them.
        (package--compile new-desc)
        ;; After compilation, load again any files loaded by
        ;; `activate-1', so that we use the byte-compiled definitions.
        (package--load-files-for-activation new-desc :reload)))
904 905
    pkg-dir))

906 907 908 909 910 911 912 913
(defun package-generate-description-file (pkg-desc pkg-file)
  "Create the foo-pkg.el file for single-file packages."
  (let* ((name (package-desc-name pkg-desc)))
    (let ((print-level nil)
          (print-quoted t)
          (print-length nil))
      (write-region
       (concat
914 915 916
        ";;; Generated package description from "
        (replace-regexp-in-string "-pkg\\.el\\'" ".el" pkg-file)
        "  -*- no-byte-compile: t -*-\n"
917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
        (prin1-to-string
         (nconc
          (list 'define-package
                (symbol-name name)
                (package-version-join (package-desc-version pkg-desc))
                (package-desc-summary pkg-desc)
                (let ((requires (package-desc-reqs pkg-desc)))
                  (list 'quote
                        ;; Turn version lists into string form.
                        (mapcar
                         (lambda (elt)
                           (list (car elt)
                                 (package-version-join (cadr elt))))
                         requires))))
          (package--alist-to-plist-args
           (package-desc-extras pkg-desc))))
        "\n")
       nil pkg-file nil 'silent))))

;;;; Autoload
937 938
(declare-function autoload-rubric "autoload" (file &optional type feature))

939 940 941
(defun package-autoload-ensure-default-file (file)
  "Make sure that the autoload file FILE exists and if not create it."
  (unless (file-exists-p file)
942 943
    (require 'autoload)
    (write-region (autoload-rubric file "package" nil) nil file nil 'silent))
944 945 946
  file)

(defvar generated-autoload-file)
947
(defvar autoload-timestamps)
948 949 950 951 952 953
(defvar version-control)

(defun package-generate-autoloads (name pkg-dir)
  (let* ((auto-name (format "%s-autoloads.el" name))
         ;;(ignore-name (concat name "-pkg.el"))
         (generated-autoload-file (expand-file-name auto-name pkg-dir))
954 955
         ;; We don't need 'em, and this makes the output reproducible.
         (autoload-timestamps nil)
956
         ;; Silence `autoload-generate-file-autoloads'.
957
         (noninteractive inhibit-message)
958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
         (backup-inhibited t)
         (version-control 'never))
    (package-autoload-ensure-default-file generated-autoload-file)
    (update-directory-autoloads pkg-dir)
    (let ((buf (find-buffer-visiting generated-autoload-file)))
      (when buf (kill-buffer buf)))
    auto-name))

(defun package--make-autoloads-and-stuff (pkg-desc pkg-dir)
  "Generate autoloads, description file, etc.. for PKG-DESC installed at PKG-DIR."
  (package-generate-autoloads (package-desc-name pkg-desc) pkg-dir)
  (let ((desc-file (expand-file-name (package--description-file pkg-dir)
                                     pkg-dir)))
    (unless (file-exists-p desc-file)
      (package-generate-description-file pkg-desc desc-file)))
  ;; FIXME: Create foo.info and dir file from foo.texi?
  )

;;;; Compilation
977
(defvar warning-minimum-level)
978
(defun package--compile (pkg-desc)
979 980 981
  "Byte-compile installed package PKG-DESC.
This assumes that `pkg-desc' has already been activated with
`package-activate-1'."
982
  (let ((warning-minimum-level :error)
983 984
        (save-silently inhibit-message)
        (load-path load-path))
985
    (byte-recompile-directory (package-desc-dir pkg-desc) 0 t)))
986 987 988 989 990

;;;; Inferring package from current buffer
(defun package-read-from-string (str)
  "Read a Lisp expression from STR.
Signal an error if the entire string was not used."
991 992 993 994 995 996
  (pcase-let ((`(,expr . ,offset) (read-from-string str)))
    (condition-case ()
        ;; The call to `ignore' suppresses a compiler warning.
        (progn (ignore (read-from-string str offset))
               (error "Can't read whole string"))
      (end-of-file expr))))
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017

(defun package--prepare-dependencies (deps)
  "Turn DEPS into an acceptable list of dependencies.

Any parts missing a version string get a default version string
of \"0\" (meaning any version) and an appropriate level of lists
is wrapped around any parts requiring it."
  (cond
   ((not (listp deps))
    (error "Invalid requirement specifier: %S" deps))
   (t (mapcar (lambda (dep)
                (cond
                 ((symbolp dep) `(,dep "0"))
                 ((stringp dep)
                  (error "Invalid requirement specifier: %S" dep))
                 ((and (listp dep) (null (cdr dep)))
                  (list (car dep) "0"))
                 (t dep)))
              deps))))

(declare-function lm-header "lisp-mnt" (header))
1018
(declare-function lm-homepage "lisp-mnt" (&optional file))
1019
(declare-function lm-keywords-list "lisp-mnt" (&optional file))
1020 1021
(declare-function lm-maintainer "lisp-mnt" (&optional file))
(declare-function lm-authors "lisp-mnt" (&optional file))
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034

(defun package-buffer-info ()
  "Return a `package-desc' describing the package in the current buffer.

If the buffer does not contain a conforming package, signal an
error.  If there is a package, narrow the buffer to the file's
boundaries."
  (goto-char (point-min))
  (unless (re-search-forward "^;;; \\([^ ]*\\)\\.el ---[ \t]*\\(.*?\\)[ \t]*\\(-\\*-.*-\\*-[ \t]*\\)?$" nil t)
    (error "Package lacks a file header"))
  (let ((file-name (match-string-no-properties 1))
        (desc      (match-string-no-properties 2))
        (start     (line-beginning-position)))
1035 1036
    ;; The terminating comment format could be extended to accept a
    ;; generic string that is not in English.
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
    (unless (search-forward (concat ";;; " file-name ".el ends here"))
      (error "Package lacks a terminating comment"))
    ;; Try to include a trailing newline.
    (forward-line)
    (narrow-to-region start (point))
    (require 'lisp-mnt)
    ;; Use some headers we've invented to drive the process.
    (let* ((requires-str (lm-header "package-requires"))
           ;; Prefer Package-Version; if defined, the package author
           ;; probably wants us to use it.  Otherwise try Version.
           (pkg-version
            (or (package-strip-rcs-id (lm-header "package-version"))
                (package-strip-rcs-id (lm-header "version"))))
1050
           (keywords (lm-keywords-list))
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
           (homepage (lm-homepage)))
      (unless pkg-version
        (error
            "Package lacks a \"Version\" or \"Package-Version\" header"))
      (package-desc-from-define
       file-name pkg-version desc
       (if requires-str
           (package--prepare-dependencies
            (package-read-from-string requires-str)))
       :kind 'single
1061
       :url homepage
1062
       :keywords keywords
1063 1064
       :maintainer (lm-maintainer)
       :authors (lm-authors)))))
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119

(defun package--read-pkg-desc (kind)
  "Read a `define-package' form in current buffer.
Return the pkg-desc, with desc-kind set to KIND."
  (goto-char (point-min))
  (unwind-protect
      (let* ((pkg-def-parsed (read (current-buffer)))
             (pkg-desc
              (when (eq (car pkg-def-parsed) 'define-package)
                (apply #'package-desc-from-define
                  (append (cdr pkg-def-parsed))))))
        (when pkg-desc
          (setf (package-desc-kind pkg-desc) kind)
          pkg-desc))))

(declare-function tar-get-file-descriptor "tar-mode" (file))
(declare-function tar--extract "tar-mode" (descriptor))

(defun package-tar-file-info ()
  "Find package information for a tar file.
The return result is a `package-desc'."
  (cl-assert (derived-mode-p 'tar-mode))
  (let* ((dir-name (file-name-directory
                    (tar-header-name (car tar-parse-info))))
         (desc-file (package--description-file dir-name))
         (tar-desc (tar-get-file-descriptor (concat dir-name desc-file))))
    (unless tar-desc
      (error "No package descriptor file found"))
    (with-current-buffer (tar--extract tar-desc)
      (unwind-protect
          (or (package--read-pkg-desc 'tar)
              (error "Can't find define-package in %s"
                (tar-header-name tar-desc)))
        (kill-buffer (current-buffer))))))

(defun package-dir-info ()
  "Find package information for a directory.
The return result is a `package-desc'."
  (cl-assert (derived-mode-p 'dired-mode))
  (let* ((desc-file (package--description-file default-directory)))
    (if (file-readable-p desc-file)
        (with-temp-buffer
          (insert-file-contents desc-file)
          (package--read-pkg-desc 'dir))
      (let ((files (directory-files default-directory t "\\.el\\'" t))
            info)
        (while files
          (with-temp-buffer
            (insert-file-contents (pop files))
            ;; When we find the file with the data,
            (when (setq info (ignore-errors (package-buffer-info)))
              ;; stop looping,
              (setq files nil)
              ;; set the 'dir kind,
              (setf (package-desc-kind info) 'dir))))
1120 1121
        (unless info
          (error "No .el files with package headers in `%s'" default-directory))
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
        ;; and return the info.
        info))))


;;; Communicating with Archives
;; Set of low-level functions for communicating with archives and
;; signature checking.
(defun package--write-file-no-coding (file-name)
  (let ((buffer-file-coding-system 'no-conversion))
    (write-region (point-min) (point-max) file-name nil 'silent)))

(declare-function url-http-file-exists-p "url-http" (url))
1134

1135 1136 1137
(defun package--archive-file-exists-p (location file)
  (let ((http (string-match "\\`https?:" location)))
    (if http
1138 1139 1140
        (progn
          (require 'url-http)
          (url-http-file-exists-p (concat location file)))
1141 1142 1143
      (file-exists-p (expand-file-name file location)))))

(declare-function epg-make-context "epg"
1144 1145 1146 1147
                  (&optional protocol armor textmode include-certs
                             cipher-algorithm
                             digest-algorithm
                             compress-algorithm))
1148
(declare-function epg-verify-string "epg" (context signature
1149
                                                   &optional signed-text))
1150
(declare-function epg-context-result-for "epg" (context name))
1151
(declare-function epg-signature-status "epg" (signature) t)
1152 1153
(declare-function epg-signature-to-string "epg" (signature))

1154 1155 1156 1157
(defun package--display-verify-error (context sig-file)
  (unless (equal (epg-context-error-output context) "")
    (with-output-to-temp-buffer "*Error*"
      (with-current-buffer standard-output
1158 1159 1160 1161 1162 1163 1164
        (if (epg-context-result-for context 'verify)
            (insert (format "Failed to verify signature %s:\n" sig-file)
                    (mapconcat #'epg-signature-to-string
                               (epg-context-result-for context 'verify)
                               "\n"))
          (insert (format "Error while verifying signature %s:\n" sig-file)))
        (insert "\nCommand output:\n" (epg-context-error-output context))))))
1165

1166 1167 1168 1169 1170 1171 1172 1173 1174
(defmacro package--with-work-buffer (location file &rest body)
  "Run BODY in a buffer containing the contents of FILE at LOCATION.
LOCATION is the base location of a package archive, and should be
one of the URLs (or file names) specified in `package-archives'.
FILE is the name of a file relative to that base location.

This macro retrieves FILE from LOCATION into a temporary buffer,
and evaluates BODY while that buffer is current.  This work
buffer is killed afterwards.  Return the last value in BODY."
1175 1176
  (declare (indent 2) (debug t)
           (obsolete package--with-response-buffer "25.1"))
1177 1178 1179 1180 1181 1182 1183 1184 1185
  `(with-temp-buffer
     (if (string-match-p "\\`https?:" ,location)
         (url-insert-file-contents (concat ,location ,file))
       (unless (file-name-absolute-p ,location)
         (error "Archive location %s is not an absolute file name"
           ,location))
       (insert-file-contents (expand-file-name ,file ,location)))
     ,@body))

1186 1187 1188 1189 1190 1191
(cl-defmacro package--with-response-buffer (url &rest body &key async file error-form noerror &allow-other-keys)
  "Access URL and run BODY in a buffer containing the response.
Point is after the headers when BODY runs.
FILE, if provided, is added to URL.
URL can be a local file name, which must be absolute.
ASYNC, if non-nil, runs the request asynchronously.
1192 1193 1194
ERROR-FORM is run only if a connection error occurs.  If NOERROR
is non-nil, don't propagate connection errors (does not apply to
errors signaled by ERROR-FORM or by BODY).
1195 1196 1197 1198 1199

\(fn URL &key ASYNC FILE ERROR-FORM NOERROR &rest BODY)"
  (declare (indent defun) (debug t))
  (while (keywordp (car body))
    (setq body (cdr (cdr body))))
1200 1201
  (macroexp-let2* nil ((url-1 url)
                       (noerror-1 noerror))
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
    (let ((url-sym (make-symbol "url"))
          (b-sym (make-symbol "b-sym")))
      `(cl-macrolet ((unless-error (body-2 &rest before-body)
                                   (let ((err (make-symbol "err")))
                                     `(with-temp-buffer
                                        (when (condition-case ,err
                                                  (progn ,@before-body t)
                                                ,(list 'error ',error-form
                                                       (list 'unless ',noerror-1
                                                             `(signal (car ,err) (cdr ,err)))))
                                          ,@body-2)))))
         (if (string-match-p "\\`https?:" ,url-1)
             (let ((,url-sym (concat ,url-1 ,file)))
               (if ,async
                   (unless-error nil
                                 (url-retrieve ,url-sym
                                               (lambda (status)
                                                 (let ((,b-sym (current-buffer)))
                                                   (require 'url-handlers)
                                                   (unless-error ,body
1222
                                                                 (when-let* ((er (plist-get status :error)))
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
                                                                   (error "Error retrieving: %s %S" ,url-sym er))
                                                                 (with-current-buffer ,b-sym
                                                                   (goto-char (point-min))
                                                                   (unless (search-forward-regexp "^\r?\n\r?" nil 'noerror)
                                                                     (error "Error retrieving: %s %S" ,url-sym "incomprehensible buffer")))
                                                                 (url-insert-buffer-contents ,b-sym ,url-sym)
                                                                 (kill-buffer ,b-sym)
                                                                 (goto-char (point-min)))))
                                               nil
                                               'silent))
                 (unless-error ,body (url-insert-file-contents ,url-sym))))
           (unless-error ,body
                         (let ((url (expand-file-name ,file ,url-1)))
                           (unless (file-name-absolute-p url)
                             (error "Location %s is not a url nor an absolute file name" url))
                           (insert-file-contents url))))))))
1239 1240

(define-error 'bad-signature "Failed to verify signature")
1241 1242 1243 1244 1245

(defun package--check-signature-content (content string &optional sig-file)
  "Check signature CONTENT against STRING.
SIG-FILE is the name of the signature file, used when signaling
errors."
1246 1247 1248
  (let ((context (epg-make-context 'OpenPGP)))
    (when package-gnupghome-dir
      (setf (epg-context-home-directory context) package-gnupghome-dir))
1249
    (condition-case error
1250 1251
        (epg-verify-string context content string)
      (error (package--display-verify-error context sig-file)
1252
             (signal 'bad-signature error)))
1253 1254 1255 1256
    (let (good-signatures had-fatal-error)
      ;; The .sig file may contain multiple signatures.  Success if one
      ;; of the signatures is good.
      (dolist (sig (epg-context-result-for context 'verify))
1257 1258 1259 1260 1261 1262 1263 1264 1265
        (if (eq (epg-signature-status sig) 'good)
            (push sig good-signatures)
          ;; If package-check-signature is allow-unsigned, don't
          ;; signal error when we can't verify signature because of
          ;; missing public key.  Other errors are still treated as
          ;; fatal (bug#17625).
          (unless (and (eq package-check-signature 'allow-unsigned)
                       (eq (epg-signature-status sig) 'no-pubkey))
            (setq had-fatal-error t))))
1266 1267 1268
      (when (or (null good-signatures)
                (and (eq package-check-signature 'all)
                     had-fatal-error))
1269
        (package--display-verify-error context sig-file)
1270
        (signal 'bad-signature (list sig-file)))
1271
      good-signatures)))
1272

1273
(defun package--check-signature (location file &optional string async callback unwind)
1274
  "Check signature of the current buffer.
1275
Download the signature file from LOCATION by appending \".sig\"
1276
to FILE.
1277
GnuPG keyring location depends on `package-gnupghome-dir'.
1278 1279 1280 1281
STRING is the string to verify, it defaults to `buffer-string'.
If ASYNC is non-nil, the download of the signature file is
done asynchronously.

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
If the signature does not verify, signal an error.
If the signature is verified and CALLBACK was provided, `funcall'
CALLBACK with the list of good signatures as argument (the list
can be empty).
If no signatures file is found, and `package-check-signature' is
`allow-unsigned', call CALLBACK with a nil argument.
Otherwise, an error is signaled.

UNWIND, if provided, is a function to be called after everything
else, even if an error is signaled."
1292 1293
  (let ((sig-file (concat file ".sig"))
        (string (or string (buffer-string))))
1294 1295
    (package--with-response-buffer location :file sig-file
      :async async :noerror t
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
      ;; Connection error is assumed to mean "no sig-file".
      :error-form (let ((allow-unsigned (eq package-check-signature 'allow-unsigned)))
                    (when (and callback allow-unsigned)
                      (funcall callback nil))
                    (when unwind (funcall unwind))
                    (unless allow-unsigned
                      (error "Unsigned file `%s' at %s" file location)))
      ;; OTOH, an error here means "bad signature", which we never
      ;; suppress.  (Bug#22089)
      (unwind-protect
          (let ((sig (package--check-signature-content (buffer-substring (point) (point-max))
                                                       string sig-file)))
            (when callback (funcall callback sig))
            sig)
        (when unwind (funcall unwind))))))
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344

;;; Packages on Archives
;; The following variables store information about packages available
;; from archives.  The most important of these is
;; `package-archive-contents' which is initially populated by the
;; function `package-read-all-archive-contents' from a cache on disk.
;; The `package-initialize' command is also closely related to this
;; section, but it has its own section.
(defconst package-archive-version 1
  "Version number of the package archive understood by this file.
Lower version numbers than this will probably be understood as well.")

;; We don't prime the cache since it tends to get out of date.
(defvar package-archive-contents nil
  "Cache of the contents of the Emacs Lisp Package Archive.
This is an alist mapping package names (symbols) to
non-empty lists of `package-desc' structures.")
(put 'package-archive-contents 'risky-local-variable t)

(defvar package--compatibility-table nil
  "Hash table connecting package names to their compatibility.
Each key is a symbol, the name of a package.

The value is either nil, representing an incompatible package, or
a version list, representing the highest compatible version of
that package which is available.

A package is considered incompatible if it requires an Emacs
version higher than the one being used.  To check for package
\(in)compatibility, don't read this table directly, use
`package--incompatible-p' which also checks dependencies.")

(defun package--build-compatibility-table ()
  "Build `package--compatibility-table' with `package--mapc'."
1345 1346
  ;; Initialize the list of built-ins.
  (require 'finder-inf nil t)
1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
  ;; Build compat table.
  (setq package--compatibility-table (make-hash-table :test 'eq))
  (package--mapc #'package--add-to-compatibility-table))

(defun package--add-to-compatibility-table (pkg)
  "If PKG is compatible (without dependencies), add to the compatibility table.
PKG is a package-desc object.
Only adds if its version is higher than what's already stored in
the table."
  (unless (package--incompatible-p pkg 'shallow)
    (let* ((name (package-desc-name pkg))
           (version (or (package-desc-version pkg) '(0)))
           (table-version (gethash name package--compatibility-table)))
      (when (or (not table-version)
                (version-list-< table-version version))
        (puthash name version package--compatibility-table)))))

;; Package descriptor objects used inside the "archive-contents" file.
;; Changing this defstruct implies changing the format of the
;; "archive-contents" files.
(cl-defstruct (package--ac-desc
               (:constructor package-make-ac-desc (version reqs summary kind extras))
               (:copier nil)
               (:type vector))
  version reqs summary kind extras)

(defun package--append-to-alist (pkg-desc alist)
  "Append an entry for PKG-DESC to the start of ALIST and return it.
This entry takes the form (`package-desc-name' PKG-DESC).

If ALIST already has an entry with this name, destructively add
PKG-DESC to the cdr of this entry instead, sorted by version
number."
  (let* ((name (package-desc-name pkg-desc))
         (priority-version (package-desc-priority-version pkg-desc))
         (existing-packages (assq name alist)))
    (if (not existing-packages)
        (cons (list name pkg-desc)
              alist)
      (while (if (and (cdr existing-packages)
                      (version-list-< priority-version
                                      (package-desc-priority-version
                                       (cadr existing-packages))))
                 (setq existing-packages (cdr existing-packages))
               (push pkg-desc (cdr existing-packages))
               nil))
      alist)))

(defun package--add-to-archive-contents (package archive)
  "Add the PACKAGE from the given ARCHIVE if necessary.
PACKAGE should have the form (NAME . PACKAGE--AC-DESC).
Also, add the originating archive to the `package-desc' structure."
  (let* ((name (car package))
         (version (package--ac-desc-version (cdr package)))
         (pkg-desc
          (package-desc-create
           :name name
           :version version
           :reqs (package--ac-desc-reqs (cdr package))
           :summary (package--ac-desc-summary (cdr package))
           :kind (package--ac-desc-kind (cdr package))
           :archive archive
           :extras (and (> (length (cdr package)) 4)
                        ;; Older archive-contents files have only 4
                        ;; elements here.
                        (package--ac-desc-extras (cdr package)))))
         (pinned-to-archive (assoc name package-pinned-packages)))
    ;; Skip entirely if pinned to another archive.
    (when (not (and pinned-to-archive
                    (not (equal (cdr pinned-to-archive) archive))))
      (setq package-archive-contents
            (package--append-to-alist pkg-desc package-archive-contents)))))

(defun package--read-archive-file (file)
  "Re-read archive file FILE, if it exists.
Will return the data from the file, or nil if the file does not exist.
Will throw an error if the archive version is too new."
  (let ((filename (expand-file-name file package-user-dir)))
    (when (file-exists-p filename)
      (with-temp-buffer
1427 1428
        (let ((coding-system-for-read 'utf-8))
          (insert-file-contents filename))
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
        (let ((contents (read (current-buffer))))
          (if (> (car contents) package-archive-version)
              (error "Package archive version %d is higher than %d"
                (car contents) package-archive-version))
          (cdr contents))))))

(defun package-read-archive-contents (archive)
  "Re-read archive contents for ARCHIVE.
If successful, set the variable `package-archive-contents'.
If the archive version is too new, signal an error."
  ;; Version 1 of 'archive-contents' is identical to our internal
  ;; representation.
  (let* ((contents-file (format "archives/%s/archive-contents" archive))
         (contents (package--read-archive-file contents-file)))
    (when contents
      (dolist (package contents)
        (package--add-to-archive-contents package archive)))))

1447 1448 1449 1450 1451 1452 1453
(defvar package--old-archive-priorities nil
  "Store currently used `package-archive-priorities'.
This is the value of `package-archive-priorities' last time
`package-read-all-archive-contents' was called.  It can be used
by arbitrary functions to decide whether it is necessary to call
it again.")

1454 1455 1456 1457
(defun package-read-all-archive-contents ()
  "Re-read `archive-contents', if it exists.
If successful, set `package-archive-contents'."
  (setq package-archive-contents nil)
1458
  (setq package--old-archive-priorities package-archive-priorities)
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
  (dolist (archive package-archives)
    (package-read-archive-contents (car archive))))

;;;; Package Initialize
;; A bit of a milestone.  This brings together some of the above
;; sections and populates all relevant lists of packages from contents
;; available on disk.
(defvar package--initialized nil)

;;;###autoload
(defun package-initialize (&optional no-activate)
  "Load Emacs Lisp packages, and activate them.
The variable `package-load-list' controls which packages to load.
1472
If optional arg NO-ACTIVATE is non-nil, don't activate packages.
1473 1474
If called as part of loading `user-init-file', set
`package-enable-at-startup' to nil, to prevent accidentally
1475
loading packages twice.
1476

1477 1478
It is not necessary to adjust `load-path' or `require' the
individual packages after calling `package-initialize' -- this is
1479 1480 1481 1482 1483 1484 1485 1486
taken care of by `package-initialize'.

If `package-initialize' is called twice during Emacs startup,
signal a warning, since this is a bad idea except in highly
advanced use cases.  To suppress the warning, remove the
superfluous call to `package-initialize' from your init-file.  If
you have code which must run before `package-initialize', put
that code in the early init-file."
1487
  (interactive)
1488 1489 1490
  (when (and package--initialized (not after-init-time))
    (lwarn '(package reinitialization) :warning
           "Unnecessary call to `package-initialize' in init file"))
1491
  (setq package-alist nil)
1492
  (setq package-enable-at-startup nil)
1493 1494
  (package-load-all-descriptors)
  (package-read-all-archive-contents)
1495
  (setq package--initialized t)
1496
  (unless no-activate
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
    (package-activate-all))
  ;; This uses `package--mapc' so it must be called after
  ;; `package--initialized' is t.
  (package--build-compatibility-table))

(defvar package-quickstart-file)

;;;###autoload
(defun package-activate-all ()
  "Activate all installed packages.
The variable `package-load-list' controls which packages to load."
  (setq package-enable-at-startup nil)
  (if (file-readable-p package-quickstart-file)
      ;; Skip load-source-file-function which would slow us down by a factor
      ;; 2 (this assumes we were careful to save this file so it doesn't need
      ;; any decoding).
      (let ((load-source-file-function nil))
1514
        (load package-quickstart-file nil 'nomessage))
1515
    (dolist (elt (package--alist))
1516 1517 1518 1519
      (condition-case err
          (package-activate (car elt))
        ;; Don't let failure of activation of a package arbitrarily stop
        ;; activation of further packages.
1520
        (error (message "%s" (error-message-string err)))))))
1521 1522 1523 1524

;;;; Populating `package-archive-contents' from archives
;; This subsection populates the variables listed above from the
;; actual archives, instead of from a local cache.
1525 1526
(defvar package--downloads-in-progress nil
  "List of in-progress asynchronous downloads.")
1527

1528
(declare-function epg-import-keys-from-file "epg" (context keys))
1529

1530 1531 1532 1533 1534
;;;###autoload
(defun package-import-keyring (&optional file)
  "Import keys from FILE."
  (interactive "fFile: ")
  (setq file (expand-file-name file))
1535 1536 1537 1538 1539
  (let ((context (epg-make-context 'OpenPGP)))
    (when package-gnupghome-dir
      (with-file-modes 448
        (make-directory package-gnupghome-dir t))
      (setf (epg-context-home-directory context) package-gnupghome-dir))
1540
    (message "Importing %s..." (file-name-nondirectory file))
1541
    (epg-import-keys-from-file context file)
1542
    (message "Importing %s...done" (file-name-nondirectory file))))
1543

1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558
(defvar package--post-download-archives-hook nil
  "Hook run after the archive contents are downloaded.
Don't run this hook directly.  It is meant to be run as part of
`package--update-downloads-in-progress'.")
(put 'package--post-download-archives-hook 'risky-local-variable t)

(defun package--update-downloads-in-progress (entry)
  "Remove ENTRY from `package--downloads-in-progress'.
Once it's empty, run `package--post-download-archives-hook'."
  ;; Keep track of the downloading progress.
  (setq package--downloads-in-progress
        (remove entry package--downloads-in-progress))
  ;; If this was the last download, run the hook.
  (unless package--downloads-in-progress
    (package-read-all-archive-contents)
1559
    (package--build-compatibility-table)
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
    ;; We message before running the hook, so the hook can give
    ;; messages as well.
    (message "Package refresh done")
    (run-hooks 'package--post-download-archives-hook)))

(defun package--download-one-archive (archive file &optional async)
  "Retrieve an archive file FILE from ARCHIVE, and cache it.
ARCHIVE should be a cons cell of the form (NAME . LOCATION),
similar to an entry in `package-alist'.  Save the cached copy to
\"archives/NAME/FILE\" in `package-user-dir'."
1570 1571 1572
  (package--with-response-buffer (cdr archive) :file file
    :async async
    :error-form (package--update-downloads-in-progress archive)
1573 1574 1575
    (let* ((location (cdr archive))
           (name (car archive))
           (content (buffer-string))
1576
           (dir (expand-file-name (concat "archives/" name) package-user-dir))
1577
           (local-file (expand-file-name file dir)))
1578
      (when (listp (read content))
1579 1580
        (make-directory dir t)
        (if (or (not package-check-signature)
1581
                (member name package-unsigned-archives))
1582 1583
            ;; If we don't care about the signature, save the file and
            ;; we're done.
1584 1585
            (progn (let ((coding-system-for-write 'utf-8))
                     (write-region content nil local-file nil 'silent))
1586 1587 1588 1589
                   (package--update-downloads-in-progress archive))
          ;; If we care, check it (perhaps async) and *then* write the file.
          (package--check-signature
           location file content async
1590
           ;; This function will be called after signature checking.
1591
           (lambda (&optional good-sigs)
1592 1593
             (let ((coding-system-for-write 'utf-8))
               (write-region content nil local-file nil 'silent))
1594 1595 1596
             ;; Write out good signatures into archive-contents.signed file.
             (when good-sigs
               (write-region (mapconcat #'epg-signature-to-string good-sigs "\n")