char-fold.el 11.9 KB
Newer Older
1
;;; char-fold.el --- match unicode to similar ASCII -*- lexical-binding: t; -*-
2

Paul Eggert's avatar
Paul Eggert committed
3
;; Copyright (C) 2015-2019 Free Software Foundation, Inc.
4

5
;; Maintainer: emacs-devel@gnu.org
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
;; Keywords: matching

;; This file is part of GNU Emacs.

;; GNU Emacs is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
21
;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
22 23 24

;;; Code:

25
(eval-and-compile (put 'char-fold-table 'char-table-extra-slots 1))
26

27 28 29 30 31
(eval-and-compile
  (defun char-fold-make-table ()
    (let* ((equiv (make-char-table 'char-fold-table))
           (equiv-multi (make-char-table 'char-fold-table))
           (table (unicode-property-table-internal 'decomposition)))
32 33
      (set-char-table-extra-slot equiv 0 equiv-multi)

34
      ;; Ensure the table is populated.
35 36 37 38 39
      (let ((func (char-table-extra-slot table 1)))
        (map-char-table (lambda (char v)
                          (when (consp char)
                            (funcall func (car char) v table)))
                        table))
40

41 42
      ;; Compile a list of all complex characters that each simple
      ;; character should match.
43 44 45 46 47 48
      ;; In summary this loop does 3 things:
      ;; - A complex character might be allowed to match its decomp.
      ;; - The decomp is allowed to match the complex character.
      ;; - A single char of the decomp might be allowed to match the
      ;;   character.
      ;; Some examples in the comments below.
49
      (map-char-table
50 51
       (lambda (char decomp)
         (when (consp decomp)
Paul Eggert's avatar
Paul Eggert committed
52
           ;; Skip trivial cases like ?a decomposing to (?a).
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
           (unless (and (not (cdr decomp))
                        (eq char (car decomp)))
             (if (symbolp (car decomp))
                 ;; Discard a possible formatting tag.
                 (setq decomp (cdr decomp))
               ;; If there's no formatting tag, ensure that char matches
               ;; its decomp exactly.  This is because we want 'ä' to
               ;; match 'ä', but we don't want '¹' to match '1'.
               (aset equiv char
                     (cons (apply #'string decomp)
                           (aref equiv char))))

             ;; Allow the entire decomp to match char.  If decomp has
             ;; multiple characters, this is done by adding an entry
             ;; to the alist of the first character in decomp.  This
             ;; allows 'ff' to match 'ff', 'ä' to match 'ä', and '1' to
             ;; match '¹'.
             (let ((make-decomp-match-char
                    (lambda (decomp char)
                      (if (cdr decomp)
                          (aset equiv-multi (car decomp)
                                (cons (cons (apply #'string (cdr decomp))
                                            (regexp-quote (string char)))
                                      (aref equiv-multi (car decomp))))
                        (aset equiv (car decomp)
                              (cons (char-to-string char)
                                    (aref equiv (car decomp))))))))
               (funcall make-decomp-match-char decomp char)
               ;; Do it again, without the non-spacing characters.
               ;; This allows 'a' to match 'ä'.
               (let ((simpler-decomp nil)
                     (found-one nil))
                 (dolist (c decomp)
                   (if (> (get-char-code-property c 'canonical-combining-class) 0)
                       (setq found-one t)
                     (push c simpler-decomp)))
                 (when (and simpler-decomp found-one)
                   (funcall make-decomp-match-char simpler-decomp char)
                   ;; Finally, if the decomp only had one spacing
                   ;; character, we allow this character to match the
                   ;; decomp.  This is to let 'a' match 'ä'.
                   (unless (cdr simpler-decomp)
                     (aset equiv (car simpler-decomp)
                           (cons (apply #'string decomp)
                                 (aref equiv (car simpler-decomp)))))))))))
98 99 100
       table)

      ;; Add some manual entries.
101 102
      (dolist (it '((?\" """ "“" "”" "”" "„" "⹂" "〞" "‟" "‟" "❞" "❝" "❠" "“" "„" "〝" "〟" "🙷" "🙶" "🙸" "«" "»")
                    (?' "❟" "❛" "❜" "‘" "’" "‚" "‛" "‚" "󠀢" "❮" "❯" "‹" "›")
103
                    (?` "❛" "‘" "‛" "󠀢" "❮" "‹")))
104 105 106
        (let ((idx (car it))
              (chars (cdr it)))
          (aset equiv idx (append chars (aref equiv idx)))))
107 108

      ;; Convert the lists of characters we compiled into regexps.
109
      (map-char-table
110 111 112
       (lambda (char decomp-list)
         (let ((re (regexp-opt (cons (char-to-string char) decomp-list))))
           (if (consp char) ; FIXME: char never is consp?
113 114
               (set-char-table-range equiv char re)
             (aset equiv char re))))
115
       equiv)
116 117 118 119 120
      equiv)))

(defconst char-fold-table
  (eval-when-compile
    (char-fold-make-table))
121
  "Used for folding characters of the same group during search.
122
This is a char-table with the `char-fold-table' subtype.
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

Let us refer to the character in question by char-x.
Each entry is either nil (meaning char-x only matches literally)
or a regexp.  This regexp should match anything that char-x can
match by itself \(including char-x).  For instance, the default
regexp for the ?+ character is \"[+⁺₊﬩﹢+]\".

This table also has one extra slot which is also a char-table.
Each entry in the extra slot is an alist used for multi-character
matching (which may be nil).  The elements of the alist should
have the form (SUFFIX . OTHER-REGEXP).  If the characters after
char-x are equal to SUFFIX, then this combination of char-x +
SUFFIX is allowed to match OTHER-REGEXP.  This is in addition to
char-x being allowed to match REGEXP.
For instance, the default alist for ?f includes:
    \((\"fl\" . \"ffl\") (\"fi\" . \"ffi\")
     (\"i\" . \"fi\") (\"f\" . \"ff\"))

Exceptionally for the space character (32), ALIST is ignored.")
142

143
(defun char-fold--make-space-string (n)
144 145 146 147
  "Return a string that matches N spaces."
  (format "\\(?:%s\\|%s\\)"
          (make-string n ?\s)
          (apply #'concat
148
                 (make-list n (or (aref char-fold-table ?\s) " ")))))
149

150
;;;###autoload
151
(defun char-fold-to-regexp (string &optional lax from)
152
  "Return a regexp matching anything that char-folds into STRING.
153
Any character in STRING that has an entry in
154
`char-fold-table' is replaced with that entry (which is a
155 156
regexp) and other characters are `regexp-quote'd.

157 158 159 160 161 162
When LAX is non-nil, then the final character also matches ligatures
partially, for instance, the search string \"f\" will match \"fi\",
so when typing the search string in isearch while the cursor is on
a ligature, the search won't try to immediately advance to the next
complete match, but will stay on the partially matched ligature.

163 164
If the resulting regexp would be too long for Emacs to handle,
just return the result of calling `regexp-quote' on STRING.
165

166 167
FROM is for internal use.  It specifies an index in the STRING
from which to start."
168
  (let* ((spaces 0)
169
         (multi-char-table (char-table-extra-slot char-fold-table 0))
170 171 172
         (i (or from 0))
         (end (length string))
         (out nil))
173 174 175 176 177 178 179 180
    ;; When the user types a space, we want to match the table entry
    ;; for ?\s, which is generally a regexp like "[ ...]".  However,
    ;; the `search-spaces-regexp' variable doesn't "see" spaces inside
    ;; these regexp constructs, so we need to use "\\( \\|[ ...]\\)"
    ;; instead (to manually expose a space).  Furthermore, the lax
    ;; search engine acts on a bunch of spaces, not on individual
    ;; spaces, so if the string contains sequential spaces like "  ", we
    ;; need to keep them grouped together like this: "\\(  \\|[ ...][ ...]\\)".
181 182
    (while (< i end)
      (pcase (aref string i)
183
        (?\s (setq spaces (1+ spaces)))
184
        (c (when (> spaces 0)
185
             (push (char-fold--make-space-string spaces) out)
186
             (setq spaces 0))
187
           (let ((regexp (or (aref char-fold-table c)
188
                             (regexp-quote (string c))))
189 190 191
                 ;; Long string.  The regexp would probably be too long.
                 (alist (unless (> end 50)
                          (aref multi-char-table c))))
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
             (push (if (and lax alist (= (1+ i) end))
                       (concat "\\(?:" regexp "\\|"
                               (mapconcat (lambda (entry)
                                            (cdr entry)) alist "\\|") "\\)")
                     (let ((matched-entries nil)
                           (max-length 0))
                       (dolist (entry alist)
                         (let* ((suffix (car entry))
                                (len-suf (length suffix)))
                           (when (eq (compare-strings suffix 0 nil
                                                      string (1+ i) (+ i 1 len-suf)
                                                      nil)
                                     t)
                             (push (cons len-suf (cdr entry)) matched-entries)
                             (setq max-length (max max-length len-suf)))))
                       ;; If no suffixes matched, just go on.
                       (if (not matched-entries)
                           regexp
210 211 212
;;; If N suffixes match, we "branch" out into N+1 executions for the
;;; length of the longest match.  This means "fix" will match "fix" but
;;; not "fⅸ", but it's necessary to keep the regexp size from scaling
213
;;; exponentially.  See https://lists.gnu.org/r/emacs-devel/2015-11/msg02562.html
214 215 216 217 218 219 220 221 222 223 224 225
                         (let ((subs (substring string (1+ i) (+ i 1 max-length))))
                           ;; `i' is still going to inc by 1 below.
                           (setq i (+ i max-length))
                           (concat
                            "\\(?:"
                            (mapconcat (lambda (entry)
                                         (let ((length (car entry))
                                               (suffix-regexp (cdr entry)))
                                           (concat suffix-regexp
                                                   (char-fold-to-regexp subs nil length))))
                                       `((0 . ,regexp) . ,matched-entries) "\\|")
                            "\\)")))))
226
                   out))))
227 228
      (setq i (1+ i)))
    (when (> spaces 0)
229
      (push (char-fold--make-space-string spaces) out))
230
    (let ((regexp (apply #'concat (nreverse out))))
231
      ;; Limited by `MAX_BUF_SIZE' in `regex-emacs.c'.
232
      (if (> (length regexp) 5000)
233 234
          (regexp-quote string)
        regexp))))
235 236 237


;;; Commands provided for completeness.
238 239 240
(defun char-fold-search-forward (string &optional bound noerror count)
  "Search forward for a char-folded version of STRING.
STRING is converted to a regexp with `char-fold-to-regexp',
241 242 243
which is searched for with `re-search-forward'.
BOUND NOERROR COUNT are passed to `re-search-forward'."
  (interactive "sSearch: ")
244
  (re-search-forward (char-fold-to-regexp string) bound noerror count))
245

246 247 248
(defun char-fold-search-backward (string &optional bound noerror count)
  "Search backward for a char-folded version of STRING.
STRING is converted to a regexp with `char-fold-to-regexp',
249 250 251
which is searched for with `re-search-backward'.
BOUND NOERROR COUNT are passed to `re-search-backward'."
  (interactive "sSearch: ")
252
  (re-search-backward (char-fold-to-regexp string) bound noerror count))
253

254
(provide 'char-fold)
255

256
;;; char-fold.el ends here