newst-backend.el 99.3 KB
Newer Older
1
;;; newst-backend.el --- Retrieval backend for newsticker.
Ulf Jasper's avatar
Ulf Jasper committed
2

Glenn Morris's avatar
Glenn Morris committed
3
;; Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009
4
;;   Free Software Foundation, Inc.
Ulf Jasper's avatar
Ulf Jasper committed
5 6

;; Author:      Ulf Jasper <ulf.jasper@web.de>
7
;; Filename:    newst-backend.el
Ulf Jasper's avatar
Ulf Jasper committed
8 9
;; URL:         http://www.nongnu.org/newsticker
;; Keywords:    News, RSS, Atom
Ulf Jasper's avatar
Ulf Jasper committed
10
;; Time-stamp:  "6. Dezember 2009, 19:15:32 (ulf)"
Ulf Jasper's avatar
Ulf Jasper committed
11 12 13

;; ======================================================================

Glenn Morris's avatar
Glenn Morris committed
14 15
;; This file is part of GNU Emacs.

Ulf Jasper's avatar
Ulf Jasper committed
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
;; GNU Emacs is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.

;; ======================================================================

;;; Commentary:

;; See newsticker.el

;; ======================================================================
;;; Code:

(require 'derived)
(require 'xml)

;; Silence warnings
(defvar w3-mode-map)
(defvar w3m-minor-mode-map)


(defvar newsticker--retrieval-timer-list nil
  "List of timers for news retrieval.
This is an alist, each element consisting of (feed-name . timer).")

(defvar newsticker--download-logos nil
  "If non-nil download feed logos if available.")

(defvar newsticker--sentinel-callback nil
  "Function called at end of `newsticker--sentinel'.")

;;;###autoload
(defun newsticker-running-p ()
  "Check whether newsticker is running.
Return t if newsticker is running, nil otherwise.  Newsticker is
considered to be running if the newsticker timer list is not empty."
  (> (length newsticker--retrieval-timer-list) 0))

;; ======================================================================
;;; Customization
;; ======================================================================
(defgroup newsticker nil
  "Aggregator for RSS and Atom feeds."
  :group 'applications)

(defconst newsticker--raw-url-list-defaults
  '(("CNET News.com"
     "http://export.cnet.com/export/feeds/news/rss/1,11176,,00.xml")
    ("Debian Security Advisories"
    "http://www.debian.org/security/dsa.en.rdf")
    ("Debian Security Advisories - Long format"
    "http://www.debian.org/security/dsa-long.en.rdf")
    ("Emacs Wiki"
    "http://www.emacswiki.org/cgi-bin/wiki.pl?action=rss"
    nil
    3600)
    ("Freshmeat.net"
82
    "http://freshmeat.net/index.atom")
Ulf Jasper's avatar
Ulf Jasper committed
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
    ("Kuro5hin.org"
    "http://www.kuro5hin.org/backend.rdf")
    ("LWN (Linux Weekly News)"
    "http://lwn.net/headlines/rss")
    ("NY Times: Technology"
    "http://partners.userland.com/nytRss/technology.xml")
    ("NY Times"
    "http://partners.userland.com/nytRss/nytHomepage.xml")
    ("Quote of the day"
    "http://www.quotationspage.com/data/qotd.rss"
    "07:00"
    86400)
    ("The Register"
    "http://www.theregister.co.uk/tonys/slashdot.rdf")
    ("slashdot"
    "http://slashdot.org/index.rss"
    nil
    3600)                        ;/. will ban you if under 3600 seconds!
    ("Wired News"
    "http://www.wired.com/news_drop/netcenter/netcenter.rdf")
    ("Heise News (german)"
    "http://www.heise.de/newsticker/heise.rdf")
    ("Tagesschau (german)"
    "http://www.tagesschau.de/newsticker.rdf"
    nil
    1800)
    ("Telepolis (german)"
    "http://www.heise.de/tp/news.rdf"))
  "Default URL list in raw form.
This list is fed into defcustom via `newsticker--splicer'.")

(defun newsticker--splicer (item)
  "Convert ITEM for splicing into `newsticker-url-list-defaults'."
  (let ((result (list 'list :tag (nth 0 item) (list 'const (nth 0 item))))
        (element (cdr item)))
    (while element
      (setq result (append result (list (list 'const (car element)))))
      (setq element (cdr element)))
    result))

(defun newsticker--set-customvar-retrieval (symbol value)
  "Set retrieval related newsticker-variable SYMBOL value to VALUE.
Calls all actions which are necessary in order to make the new
value effective."
  (if (or (not (boundp symbol))
          (equal (symbol-value symbol) value))
      (set symbol value)
    ;; something must have changed
    (let ((need-restart nil)
          (new-or-changed-feeds nil)
          (removed-feeds))
      (cond ((eq symbol 'newsticker-retrieval-interval)
             (setq need-restart t))
            ((memq symbol '(newsticker-url-list-defaults newsticker-url-list))
             (dolist (elt value)
               (unless (member elt (symbol-value symbol))
                 (setq new-or-changed-feeds (cons elt new-or-changed-feeds))))
             (dolist (elt (symbol-value symbol))
               (unless (member elt value)
                 (setq removed-feeds (cons elt removed-feeds))))))
      (cond (need-restart
             (set symbol value)
             (when (newsticker-running-p)
               (message "Restarting newsticker")
               (newsticker-stop)
               (newsticker-start)))
            (t
             (dolist (feed removed-feeds)
               (message "Stopping feed `%s'" (car feed))
               (newsticker--stop-feed (car feed)))
             (dolist (feed new-or-changed-feeds)
               (message "Starting feed `%s'" (car feed))
               (newsticker--stop-feed (car feed))
               (newsticker--start-feed feed))
             (unless new-or-changed-feeds
               (when newsticker--sentinel-callback
                 (funcall newsticker--sentinel-callback)))))
      (set symbol value))))

;; ======================================================================
;; retrieval
(defgroup newsticker-retrieval nil
  "Settings for news retrieval."
  :group 'newsticker)

(defcustom newsticker-url-list-defaults
 '(("Emacs Wiki"
    "http://www.emacswiki.org/cgi-bin/wiki.pl?action=rss"
    nil
    3600))
  "A customizable list of news feeds to select from.
These were mostly extracted from the Radio Community Server at
http://subhonker6.userland.com/rcsPublic/rssHotlist.

You may add other entries in `newsticker-url-list'."
  :type `(set ,@(mapcar `newsticker--splicer
                        newsticker--raw-url-list-defaults))
  :set 'newsticker--set-customvar-retrieval
  :group 'newsticker-retrieval)

(defcustom newsticker-url-list nil
  "The news feeds which you like to watch.

This alist will be used in addition to selection made customizing
`newsticker-url-list-defaults'.

This is an alist.  Each element consists of two items: a LABEL and a URL,
optionally followed by a START-TIME, INTERVAL specifier and WGET-ARGUMENTS.

The LABEL gives the name of the news feed.  It can be an arbitrary string.

The URL gives the location of the news feed.  It must point to a valid
RSS or Atom file.  The file is retrieved by calling wget, or whatever you
specify as `newsticker-wget-name'.

URL may also be a function which returns news data.  In this case
`newsticker-retrieval-method' etc.  are ignored for this feed.

The START-TIME can be either a string, or nil.  If it is a string it
specifies a fixed time at which this feed shall be retrieved for the
first time.  (Examples: \"11:00pm\", \"23:00\".)  If it is nil (or
unspecified), this feed will be retrieved immediately after calling
`newsticker-start'.

The INTERVAL specifies the time between retrievals for this feed.  If it
is nil (or unspecified) the default interval value as set in
`newsticker-retrieval-interval' is used.

\(newsticker.el calls `run-at-time'. The newsticker-parameters START-TIME
and INTERVAL correspond to the `run-at-time'-parameters TIME and REPEAT.)

WGET-ARGUMENTS specifies arguments for wget (see `newsticker-wget-name')
which apply for this feed only, overriding the value of
`newsticker-wget-arguments'."
  :type '(repeat (list :tag "News feed"
                       (string :tag "Label")
                       (choice :tag "URI"
                               (string :tag "String")
                               (function :tag "Function"))
                       (choice :tag "Start"
                               (const   :tag "Default" nil)
                               (string  :tag "Fixed Time"))
                       (choice :tag "Interval"
                               (const   :tag "Default" nil)
                               (const   :tag "Hourly" 3600)
                               (const   :tag "Daily" 86400)
                               (const   :tag "Weekly" 604800)
                               (integer :tag "Interval"))
                       (choice :tag "Wget Arguments"
                               (const  :tag "Default arguments" nil)
                               (repeat :tag "Special arguments" string))))
  :set 'newsticker--set-customvar-retrieval
  :group 'newsticker-retrieval)

(defcustom newsticker-retrieval-method
  'intern
  "Method for retrieving news from the web, either `intern' or `extern'.
Default value `intern' uses Emacs' built-in asynchronous download
capabilities ('url-retrieve').  If set to `extern' the external
program wget is used, see `newsticker-wget-name'."
  :type '(choice :tag "Method"
                 (const :tag "Intern" intern)
                 (const :tag "Extern" extern))
  :group 'newsticker-retrieval)

(defcustom newsticker-wget-name
  "wget"
  "Name of the program which is called to retrieve news from the web.
The canonical choice is wget but you may take any other program which is
able to return the contents of a news feed file on stdout."
  :type 'string
  :group 'newsticker-retrieval)

(defcustom newsticker-wget-arguments
  '("-q" "-O" "-")
  "Arguments which are passed to wget.
There is probably no reason to change the default settings, unless you
are living behind a firewall."
  :type '(repeat (string :tag "Argument"))
  :group 'newsticker-retrieval)

(defcustom newsticker-retrieval-interval
  3600
  "Time interval for retrieving new news items (seconds).
If this value is not positive (i.e. less than or equal to 0)
items are retrieved only once!
Please note that some feeds, e.g. Slashdot, will ban you if you
make it less than 1800 seconds (30 minutes)!"
  :type '(choice :tag "Interval"
                 (const   :tag "No automatic retrieval" 0)
                 (const   :tag "Hourly" 3600)
                 (const   :tag "Daily" 86400)
                 (const   :tag "Weekly" 604800)
                 (integer :tag "Interval"))
  :set 'newsticker--set-customvar-retrieval
  :group 'newsticker-retrieval)

(defcustom newsticker-desc-comp-max
  100
  "Relevant length of headline descriptions.
This value gives the maximum number of characters which will be
taken into account when newsticker compares two headline
descriptions."
  :type 'integer
  :group 'newsticker-retrieval)

;; ======================================================================
;; headline processing
(defgroup newsticker-headline-processing nil
  "Settings for the automatic processing of headlines."
  :group 'newsticker)

(defcustom newsticker-automatically-mark-items-as-old
  t
  "Decides whether to automatically mark items as old.
If t a new item is considered as new only after its first retrieval.  As
soon as it is retrieved a second time, it becomes old.  If not t all
items stay new until you mark them as old.  This is done in the
*newsticker* buffer."
  :type 'boolean
  :group 'newsticker-headline-processing)

(defcustom newsticker-automatically-mark-visited-items-as-old
  t
  "Decides whether to automatically mark visited items as old.
If t an item is marked as old as soon as the associated link is
visited, i.e. after pressing RET or mouse2 on the item's
headline."

  :type 'boolean
  :group 'newsticker-headline-processing)

(defcustom newsticker-keep-obsolete-items
  t
  "Decides whether to keep unread items which have been removed from feed.
If t a new item, which has been removed from the feed, is kept in
the cache until it is marked as read."
  :type 'boolean
  :group 'newsticker-headline-processing)

(defcustom newsticker-obsolete-item-max-age
  (* 60 60 24)
  "Maximal age of obsolete items, in seconds.
Obsolete items which are older than this value will be silently
deleted at the next retrieval."
  :type 'integer
  :group 'newsticker-headline-processing)

(defcustom newsticker-auto-mark-filter-list
  nil
  "A list of filters for automatically marking headlines.

This is an alist of the form (FEED-NAME PATTERN-LIST).  I.e. each
element consists of a FEED-NAME a PATTERN-LIST.  Each element of
the pattern-list has the form (AGE TITLE-OR-DESCRIPTION REGEXP).
AGE must be one of the symbols 'old or 'immortal.
TITLE-OR-DESCRIPTION must be on of the symbols 'title,
'description, or 'all.  REGEXP is a regular expression, i.e. a
string.

This filter is checked after a new headline has been retrieved.
If FEED-NAME matches the name of the corresponding news feed, the
pattern-list is checked: The new headline will be marked as AGE
if REGEXP matches the headline's TITLE-OR-DESCRIPTION.

If, for example, `newsticker-auto-mark-filter-list' looks like
 \((slashdot ('old 'title \"^Forget me!$\") ('immortal 'title \"Read me\")
  \('immortal 'all \"important\"))))

then all articles from slashdot are marked as old if they have
the title \"Forget me!\".  All articles with a title containing
the string \"Read me\" are marked as immortal.  All articles which
contain the string \"important\" in their title or their
description are marked as immortal."
  :type '(repeat (list :tag "Auto mark filter"
                       (string :tag "Feed name")
                       (repeat
                        (list :tag "Filter element"
                              (choice
                               :tag "Auto-assigned age"
                               (const :tag "Old" old)
                               (const :tag "Immortal" immortal))
                              (choice
                               :tag "Title/Description"
                               (const :tag "Title" title)
                               (const :tag "Description" description)
                               (const :tag "All" all))
                              (string :tag "Regexp")))))
  :group 'newsticker-headline-processing)

;; ======================================================================
;; hooks
(defgroup newsticker-hooks nil
  "Settings for newsticker hooks."
  :group 'newsticker)

(defcustom newsticker-start-hook
  nil
  "Hook run when starting newsticker.
This hook is run at the very end of `newsticker-start'."
  :options '(newsticker-start-ticker)
  :type 'hook
  :group 'newsticker-hooks)

(defcustom newsticker-stop-hook
  nil
  "Hook run when stopping newsticker.
This hook is run at the very end of `newsticker-stop'."
  :options nil
  :type 'hook
  :group 'newsticker-hooks)

(defcustom newsticker-new-item-functions
  nil
  "List of functions run after a new headline has been retrieved.
Each function is called with the following three arguments:
FEED  the name of the corresponding news feed,
TITLE the title of the headline,
DESC  the decoded description of the headline.

See `newsticker-download-images', and
`newsticker-download-enclosures' for sample functions.

Please note that these functions are called only once for a
headline after it has been retrieved for the first time."
  :type 'hook
  :options '(newsticker-download-images
             newsticker-download-enclosures)
  :group 'newsticker-hooks)

;; ======================================================================
;; miscellaneous
(defgroup newsticker-miscellaneous nil
  "Miscellaneous newsticker settings."
  :group 'newsticker)

(defcustom newsticker-cache-filename
  "~/.newsticker-cache"
  "Name of the newsticker cache file."
  :type 'string
  :group 'newsticker-miscellaneous)
424
(make-obsolete 'newsticker-cache-filename 'newsticker-dir "23.1")
Ulf Jasper's avatar
Ulf Jasper committed
425

Ulf Jasper's avatar
Ulf Jasper committed
426
(defcustom newsticker-dir
427 428 429
  (locate-user-emacs-file "newsticker/" ".newsticker/")
  "Directory where newsticker saves data."
  :type 'directory
Ulf Jasper's avatar
Ulf Jasper committed
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
  :group 'newsticker-miscellaneous)

;; debugging
(defcustom newsticker-debug
  nil
  "Enables some features needed for debugging newsticker.el.

If set to t newsticker.el will print lots of debugging messages, and the
buffers *newsticker-wget-<feed>* will not be closed."
  :type 'boolean
  :group 'newsticker-miscellaneous)

;; ======================================================================
;;; Compatibility section, XEmacs, Emacs
;; ======================================================================
445 446 447 448

;; FIXME It is bad practice to define compat functions with such generic names.

;; This is not needed in Emacs >= 22.1.
Ulf Jasper's avatar
Ulf Jasper committed
449 450 451
(unless (fboundp 'time-add)
  (require 'time-date);;FIXME
  (defun time-add (t1 t2)
452 453
    (with-no-warnings ; don't warn about obsolete time-to-seconds in 23.2
      (seconds-to-time (+ (time-to-seconds t1) (time-to-seconds t2))))))
Ulf Jasper's avatar
Ulf Jasper committed
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635

(unless (fboundp 'match-string-no-properties)
  (defalias 'match-string-no-properties 'match-string))

(when (featurep 'xemacs)
  (unless (fboundp 'replace-regexp-in-string)
    (defun replace-regexp-in-string (re rp st)
      (save-match-data ;; apparently XEmacs needs save-match-data
	(replace-in-string st re rp)))))

;; copied from subr.el
(unless (fboundp 'add-to-invisibility-spec)
  (defun add-to-invisibility-spec (arg)
    "Add elements to `buffer-invisibility-spec'.
See documentation for `buffer-invisibility-spec' for the kind of elements
that can be added."
    (if (eq buffer-invisibility-spec t)
        (setq buffer-invisibility-spec (list t)))
    (setq buffer-invisibility-spec
          (cons arg buffer-invisibility-spec))))

;; copied from subr.el
(unless (fboundp 'remove-from-invisibility-spec)
  (defun remove-from-invisibility-spec (arg)
    "Remove elements from `buffer-invisibility-spec'."
    (if (consp buffer-invisibility-spec)
        (setq buffer-invisibility-spec
              (delete arg buffer-invisibility-spec)))))

;; ======================================================================
;;; Internal variables
;; ======================================================================
(defvar newsticker--item-list nil
  "List of newsticker items.")
(defvar newsticker--item-position 0
  "Actual position in list of newsticker items.")
(defvar newsticker--prev-message "There was no previous message yet!"
  "Last message that the newsticker displayed.")
(defvar newsticker--scrollable-text ""
  "The text which is scrolled smoothly in the echo area.")
(defvar newsticker--buffer-uptodate-p nil
  "Tells whether the newsticker buffer is up to date.")
(defvar newsticker--latest-update-time (current-time)
  "The time at which the latest news arrived.")
(defvar newsticker--process-ids nil
  "List of PIDs of active newsticker processes.")

(defvar newsticker--cache nil "Cached newsticker data.
This is a list of the form

 ((label1
   (title description link time age index preformatted-contents
    preformatted-title extra-elements)
   ...)
  (label2
   (title description link time age index preformatted-contents
    preformatted-title extra-elements)
   ...)
  ...)

where LABEL is a symbol.  TITLE, DESCRIPTION, and LINK are
strings.  TIME is a time value as returned by `current-time'.
AGE is a symbol: 'new, 'old, 'immortal, and 'obsolete denote
ordinary news items, whereas 'feed denotes an item which is not a
headline but describes the feed itself.  INDEX denotes the
original position of the item -- used for restoring the original
order.  PREFORMATTED-CONTENTS and PREFORMATTED-TITLE hold the
formatted contents of the item's description and title.  This
speeds things up if HTML rendering is used, which is rather
slow.  EXTRA-ELEMENTS is an alist containing additional elements.")

(defvar newsticker--auto-narrow-to-feed nil
  "Automatically narrow to current news feed.
If non-nil only the items of the current news feed are visible.")

(defvar newsticker--auto-narrow-to-item nil
  "Automatically narrow to current news item.
If non-nil only the current headline is visible.")

(defconst newsticker--error-headline
  "[COULD NOT DOWNLOAD HEADLINES!]"
  "Title of error headline which will be inserted if news retrieval fails.")

;; ======================================================================
;;; Shortcuts
;; ======================================================================
(defsubst newsticker--title (item)
  "Return title of ITEM."
  (nth 0 item))
(defsubst newsticker--desc (item)
  "Return description of ITEM."
  (nth 1 item))
(defsubst newsticker--link (item)
  "Return link of ITEM."
  (nth 2 item))
(defsubst newsticker--time (item)
  "Return time of ITEM."
  (nth 3 item))
(defsubst newsticker--age (item)
  "Return age of ITEM."
  (nth 4 item))
(defsubst newsticker--pos (item)
  "Return position/index of ITEM."
  (nth 5 item))
(defsubst newsticker--preformatted-contents (item)
  "Return pre-formatted text of ITEM."
  (nth 6 item))
(defsubst newsticker--preformatted-title (item)
  "Return pre-formatted title of ITEM."
  (nth 7 item))
(defsubst newsticker--extra (item)
  "Return extra attributes of ITEM."
  (nth 8 item))
(defsubst newsticker--guid-to-string (guid)
  "Return string representation of GUID."
  (if (stringp guid)
      guid
    (car (xml-node-children guid))))
(defsubst newsticker--guid (item)
  "Return guid of ITEM."
  (newsticker--guid-to-string (assoc 'guid (newsticker--extra item))))
(defsubst newsticker--enclosure (item)
  "Return enclosure element of ITEM in the form \(...FIXME...\) or nil."
  (let ((enclosure (assoc 'enclosure (newsticker--extra item))))
    (if enclosure
        (xml-node-attributes enclosure))))
(defun newsticker--real-feed-name (feed)
  "Return real name of FEED."
  (catch 'name
    (mapc (lambda (item)
            (if (eq (newsticker--age item) 'feed)
                (throw 'name (newsticker--title item))))
          (cdr (newsticker--cache-get-feed feed)))
    (symbol-name feed)))


;; ======================================================================
;;; User fun
;; ======================================================================

(defun newsticker--start-feed (feed &optional do-not-complain-if-running)
  "Start retrieval timer for FEED.
If timer is running already a warning message is printed unless
DO-NOT-COMPLAIN-IF-RUNNING is not nil.  Add the started
name/timer pair to `newsticker--retrieval-timer-list'."
  (let* ((feed-name (car feed))
         (start-time (nth 2 feed))
         (interval (or (nth 3 feed)
                       newsticker-retrieval-interval))
         (timer (assoc (car feed)
                       newsticker--retrieval-timer-list)))
    (if timer
        (or do-not-complain-if-running
            (message "Timer for %s is running already!"
                     feed-name))
      (newsticker--debug-msg "Starting timer for %s: %s, %d"
                             feed-name start-time interval)
      ;; do not repeat retrieval if interval not positive
      (if (<= interval 0)
          (setq interval nil))
      ;; Suddenly XEmacs doesn't like start-time 0
      (if (or (not start-time)
              (and (numberp start-time) (= start-time 0)))
          (setq start-time 1))
      ;; (message "start-time %s" start-time)
      (setq timer (run-at-time start-time interval
                               'newsticker-get-news feed-name))
      (if interval
          (add-to-list 'newsticker--retrieval-timer-list
                       (cons feed-name timer))))))

;;;###autoload
(defun newsticker-start (&optional do-not-complain-if-running)
  "Start the newsticker.
Start the timers for display and retrieval.  If the newsticker, i.e. the
timers, are running already a warning message is printed unless
DO-NOT-COMPLAIN-IF-RUNNING is not nil.
Run `newsticker-start-hook' if newsticker was not running already."
  (interactive)
  (let ((running (newsticker-running-p)))
    ;; read old cache if it exists and newsticker is not running
    (unless running
Ulf Jasper's avatar
Ulf Jasper committed
636
      (newsticker--cache-read))
Ulf Jasper's avatar
Ulf Jasper committed
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
    ;; start retrieval timers -- one timer for each feed
    (dolist (feed (append newsticker-url-list-defaults newsticker-url-list))
      (newsticker--start-feed feed))
    (unless running
      (run-hooks 'newsticker-start-hook)
      (message "Newsticker started!"))))

(defun newsticker--stop-feed (feed-name)
  "Stop retrieval for feed FEED-NAME.
Delete the stopped name/timer pair from `newsticker--retrieval-timer-list'."
  (let ((name-and-timer (assoc feed-name newsticker--retrieval-timer-list)))
    (when name-and-timer
      (cancel-timer (cdr name-and-timer))
      (setq newsticker--retrieval-timer-list
            (delete name-and-timer newsticker--retrieval-timer-list)))))

(defun newsticker-stop ()
  "Stop the newsticker and the newsticker-ticker.
Cancel the timers for display and retrieval.  Run `newsticker-stop-hook'
if newsticker has been running."
  (interactive)
Ulf Jasper's avatar
Ulf Jasper committed
658
  (newsticker--cache-save)
Ulf Jasper's avatar
Ulf Jasper committed
659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
  (when (fboundp 'newsticker-stop-ticker) ; silence compiler warnings
    (newsticker-stop-ticker))
  (when (newsticker-running-p)
    (mapc (lambda (name-and-timer)
            (newsticker--stop-feed (car name-and-timer)))
          newsticker--retrieval-timer-list)
    (setq newsticker--retrieval-timer-list nil)
    (run-hooks 'newsticker-stop-hook)
    (message "Newsticker stopped!")))

(defun newsticker-get-all-news ()
  "Launch retrieval of news from all configured newsticker sites.
This does NOT start the retrieval timers."
  (interactive)
  ;; launch retrieval of news
  (mapc (lambda (item)
          (newsticker-get-news (car item)))
        (append newsticker-url-list-defaults newsticker-url-list)))

(defun newsticker-save-item (feed item)
  "Save FEED ITEM."
  (interactive)
  (let ((filename (read-string "Filename: "
                               (concat feed ":_"
                                       (replace-regexp-in-string
                                        " " "_" (newsticker--title item))
                                       ".html"))))
    (with-temp-buffer
      (insert (newsticker--desc item))
      (write-file filename t))))

(defun newsticker-add-url (url name)
  "Add given URL under given NAME to `newsticker-url-list'.
If URL is nil it is searched at point."
  (interactive
   (list
    (read-string "URL: "
                 (save-excursion
                   (end-of-line)
                   (and
                    (re-search-backward
                     "http://"
                     (if (> (point) (+ (point-min) 100))
                         (- (point) 100)
                       (point-min))
                     t)
                    (re-search-forward
                     "http://[-a-zA-Z0-9&/_.]*"
                     (if (< (point) (- (point-max) 200))
                         (+ (point) 200)
                       (point-max))
                     t)
                    (buffer-substring-no-properties (match-beginning 0)
                                                    (match-end 0)))))
    (read-string "Name: ")))
  (add-to-list 'newsticker-url-list (list name url nil nil nil) t)
  (customize-variable 'newsticker-url-list))

(defun newsticker-customize ()
  "Open the newsticker customization group."
  (interactive)
Ulf Jasper's avatar
Ulf Jasper committed
720
  (delete-other-windows)
Ulf Jasper's avatar
Ulf Jasper committed
721 722 723 724 725 726 727 728 729
  (customize-group "newsticker"))

;; ======================================================================
;;; Local stuff
;; ======================================================================
(defun newsticker--get-news-by-funcall (feed-name function)
  "Get news for the site FEED-NAME by calling FUNCTION.
See `newsticker-get-news'."
  (let ((buffername (concat " *newsticker-funcall-" feed-name "*")))
730
    (with-current-buffer (get-buffer-create buffername)
Ulf Jasper's avatar
Ulf Jasper committed
731 732 733 734 735 736 737 738 739
      (erase-buffer)
      (insert (string-to-multibyte (funcall function feed-name)))
      (newsticker--sentinel-work nil t feed-name function
                                 (current-buffer)))))

(defun newsticker--get-news-by-url (feed-name url)
  "Get news for the site FEED-NAME from address URL using `url-retrieve'.
See `newsticker-get-news'."
  (let ((coding-system-for-read 'no-conversion))
740
    (condition-case error-data
Ulf Jasper's avatar
Ulf Jasper committed
741
        (url-retrieve url 'newsticker--get-news-by-url-callback
742 743 744
                      (list feed-name))
          (error (message "Error retrieving news from %s: %s" feed-name
                          error-data))))
Ulf Jasper's avatar
Ulf Jasper committed
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
  (force-mode-line-update))

(defun newsticker--get-news-by-url-callback (status feed-name)
  "Callback function for `newsticker--get-news-by-url'.
STATUS is the return status as delivered by `url-retrieve', and
FEED-NAME is the name of the feed that the news were retrieved
from."
  (let ((buf (get-buffer-create (concat " *newsticker-url-" feed-name "*")))
        (result (string-to-multibyte (buffer-string))))
    (set-buffer buf)
    (erase-buffer)
    (insert result)
    ;; remove MIME header
    (goto-char (point-min))
    (search-forward "\n\n")
    (delete-region (point-min) (point))
    ;; read the rss/atom contents
    (newsticker--sentinel-work nil t feed-name "url-retrieve" (current-buffer))
    (when status
      (let ((status-type (car status))
            (status-details (cdr status)))
        (cond ((eq status-type :redirect)
               ;; don't care about redirects
               )
              ((eq status-type :error)
               (message "%s: Error while retrieving news from %s: %s: \"%s\""
                        (format-time-string "%A, %H:%M" (current-time))
                        feed-name
                        (car status-details) (cdr status-details))))))))

(defun newsticker--get-news-by-wget (feed-name url wget-arguments)
  "Get news for the site FEED-NAME from address URL using wget.
WGET-ARGUMENTS is a list of arguments for wget.
See `newsticker-get-news'."
  (let ((buffername (concat " *newsticker-wget-" feed-name "*")))
780
    (with-current-buffer (get-buffer-create buffername)
Ulf Jasper's avatar
Ulf Jasper committed
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
      (erase-buffer)
      ;; throw an error if there is an old wget-process around
      (if (get-process feed-name)
          (error "Another wget-process is running for %s" feed-name))
      ;; start wget
      (let* ((args (append wget-arguments (list url)))
             (proc (apply 'start-process feed-name buffername
                          newsticker-wget-name args)))
        (set-process-coding-system proc 'no-conversion 'no-conversion)
        (set-process-sentinel proc 'newsticker--sentinel)
        (setq newsticker--process-ids (cons (process-id proc)
                                            newsticker--process-ids))
        (force-mode-line-update)))))

(defun newsticker-get-news (feed-name)
  "Get news from the site FEED-NAME and load feed logo.
FEED-NAME must be a string which occurs as the label (i.e. the first element)
in an element of `newsticker-url-list' or `newsticker-url-list-defaults'."
  (newsticker--debug-msg "%s: Getting news for %s"
                         (format-time-string "%A, %H:%M" (current-time))
                         feed-name)
  (let* ((item (or (assoc feed-name newsticker-url-list)
                   (assoc feed-name newsticker-url-list-defaults)
                   (error
                    "Cannot get news for %s: Check newsticker-url-list"
                    feed-name)))
         (url (cadr item))
         (wget-arguments (or (car (cdr (cdr (cdr (cdr item)))))
                             newsticker-wget-arguments)))
    (if (functionp url)
        (newsticker--get-news-by-funcall feed-name url)
      (if (eq newsticker-retrieval-method 'intern)
          (newsticker--get-news-by-url feed-name url)
        (newsticker--get-news-by-wget feed-name url wget-arguments)))))

;; ======================================================================
;; Parsing
;; ======================================================================

(defun newsticker--sentinel (process event)
  "Sentinel for extracting news titles from an RDF buffer.
Argument PROCESS is the process which has just changed its state.
Argument EVENT tells what has happened to the process."
  (let ((p-status (process-status process))
        (exit-status (process-exit-status process))
        (name (process-name process))
        (command (process-command process))
        (buffer (process-buffer process)))
    (newsticker--sentinel-work event
                               (and (eq p-status 'exit)
                                    (= exit-status 0))
                               name command buffer)))

(defun newsticker--sentinel-work (event status-ok name command buffer)
  "Actually do the sentinel work.
Argument EVENT tells what has happened to the retrieval process.
Argument STATUS-OK is the final status of the retrieval process,
non-nil meaning retrieval was successful.
Argument NAME is the name of the retrieval process.
Argument COMMAND is the command of the retrieval process.
Argument BUFFER is the buffer of the retrieval process."
  (let ((time (current-time))
        (name-symbol (intern name))
        (something-was-added nil))
    ;; catch known errors (zombie processes, rubbish-xml etc.
    ;; if an error occurs the news feed is not updated!
    (catch 'oops
      (unless status-ok
        (setq newsticker--cache
              (newsticker--cache-add
               newsticker--cache
               name-symbol
               newsticker--error-headline
               (format
                (concat "%s: Newsticker could not retrieve news from %s.\n"
                        "Return status: `%s'\n"
                        "Command was `%s'")
                (format-time-string "%A, %H:%M" (current-time))
                name event command)
               ""
               (current-time)
               'new
               0 nil))
        (message "%s: Error while retrieving news from %s"
                 (format-time-string "%A, %H:%M" (current-time))
                 name)
        (throw 'oops nil))
      (let* ((coding-system 'utf-8)
             (node-list
              (save-current-buffer
                (set-buffer buffer)
                ;; a very very dirty workaround to overcome the
                ;; problems with the newest (20030621) xml.el:
                ;; remove all unnecessary whitespace
                (goto-char (point-min))
                (while (re-search-forward ">[ \t\r\n]+<" nil t)
                  (replace-match "><" nil t))
                ;; and another brutal workaround (20031105)!  For some
                ;; reason the xml parser does not like the colon in the
                ;; doctype name "rdf:RDF"
                (goto-char (point-min))
                (if (re-search-forward "<!DOCTYPE[ \t\n]+rdf:RDF" nil t)
                  (replace-match "<!DOCTYPE rdfColonRDF" nil t))
                ;; finally.... ~##^°!!!!!
                (goto-char (point-min))
                (while (search-forward "\r\n" nil t)
                  (replace-match "\n" nil t))
                ;; still more brutal workarounds (20040309)!  The xml
                ;; parser does not like doctype rss
                (goto-char (point-min))
                (if (re-search-forward "<!DOCTYPE[ \t\n]+rss[ \t\n]*>" nil t)
                  (replace-match "" nil t))
                ;; And another one (20050618)! (Fixed in GNU Emacs 22.0.50.18)
                ;; Remove comments to avoid this xml-parsing bug:
                ;; "XML files can have only one toplevel tag"
                (goto-char (point-min))
                (while (search-forward "<!--" nil t)
                  (let ((start (match-beginning 0)))
                    (unless (search-forward "-->" nil t)
                      (error "Can't find end of comment"))
                    (delete-region start (point))))
                ;; And another one (20050702)! If description is HTML
                ;; encoded and starts with a `<', wrap the whole
                ;; description in a CDATA expression.  This happened for
                ;; http://www.thefreedictionary.com/_/WoD/rss.aspx?type=quote
                (goto-char (point-min))
                (while (re-search-forward
                        "<description>\\(<img.*?\\)</description>" nil t)
                  (replace-match
                   "<description><![CDATA[ \\1 ]]></description>"))
                ;; And another one (20051123)! XML parser does not
                ;; like this: <yweather:location city="Frankfurt/Main"
                ;; region="" country="GM" />
                ;; try to "fix" empty attributes
                ;; This happened for
                ;; http://xml.weather.yahoo.com/forecastrss?p=GMXX0040&u=f
                (goto-char (point-min))
                (while (re-search-forward "\\(<[^>]*\\)=\"\"" nil t)
                  (replace-match "\\1=\" \""))
                ;;
                (set-buffer-modified-p nil)
                ;; check coding system
                (goto-char (point-min))
                (if (re-search-forward "encoding=\"\\([^\"]+\\)\""
                                       nil t)
                    (setq coding-system (intern (downcase (match-string 1))))
                  (setq coding-system
                        (condition-case nil
                              (check-coding-system coding-system)
                          (coding-system-error
                           (message
                            "newsticker.el: ignoring coding system %s for %s"
                            coding-system name)
                           nil))))
                ;; Decode if possible
                (when coding-system
                  (decode-coding-region (point-min) (point-max)
                                        coding-system))
                (condition-case errordata
                    ;; The xml parser might fail
                    ;; or the xml might be bugged
                    (xml-parse-region (point-min) (point-max))
                  (error (message "Could not parse %s: %s"
                                  (buffer-name) (cadr errordata))
                         (throw 'oops nil)))))
             (topnode (car node-list))
             (channelnode (car (xml-get-children topnode 'channel)))
             (imageurl nil))
        ;; mark all items as obsolete
        (newsticker--cache-replace-age newsticker--cache
                                       name-symbol
                                       'new 'obsolete-new)
        (newsticker--cache-replace-age newsticker--cache
                                       name-symbol
                                       'old 'obsolete-old)
        (newsticker--cache-replace-age newsticker--cache
                                       name-symbol
                                       'feed 'obsolete-old)

        ;; check Atom/RSS version and call corresponding parser
        (condition-case error-data
            (if (cond
                 ;; RSS 0.91
                 ((and (eq 'rss (xml-node-name topnode))
                       (string= "0.91" (xml-get-attribute topnode 'version)))
                  (setq imageurl (newsticker--get-logo-url-rss-0.91 topnode))
                  (newsticker--parse-rss-0.91 name time topnode))
                 ;; RSS 0.92
                 ((and (eq 'rss (xml-node-name topnode))
                       (string= "0.92" (xml-get-attribute topnode 'version)))
                  (setq imageurl (newsticker--get-logo-url-rss-0.92 topnode))
                  (newsticker--parse-rss-0.92 name time topnode))
                 ;; RSS 1.0
                 ((eq 'rdf:RDF (xml-node-name topnode))
                  (setq imageurl (newsticker--get-logo-url-rss-1.0 topnode))
                  (newsticker--parse-rss-1.0 name time topnode))
                 ;; RSS 2.0
                 ((and (eq 'rss (xml-node-name topnode))
                       (string= "2.0" (xml-get-attribute topnode 'version)))
                  (setq imageurl (newsticker--get-logo-url-rss-2.0 topnode))
                  (newsticker--parse-rss-2.0 name time topnode))
                 ;; Atom 0.3
                 ((and (eq 'feed (xml-node-name topnode))
                       (string= "http://purl.org/atom/ns#"
                                (xml-get-attribute topnode 'xmlns)))
                  (setq imageurl (newsticker--get-logo-url-atom-0.3 topnode))
                  (newsticker--parse-atom-0.3 name time topnode))
                 ;; Atom 1.0
                 ((and (eq 'feed (xml-node-name topnode))
                       (string= "http://www.w3.org/2005/Atom"
                                (xml-get-attribute topnode 'xmlns)))
                  (setq imageurl (newsticker--get-logo-url-atom-1.0 topnode))
                  (newsticker--parse-atom-1.0 name time topnode))
                 ;; unknown feed type
                 (t
                  (newsticker--debug-msg "Feed type unknown: %s: %s"
                                         (xml-node-name topnode) name)
                  nil))
                (setq something-was-added t))
1000
          (error (message "sentinelerror in %s: %s" name error-data)))
Ulf Jasper's avatar
Ulf Jasper committed
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031

        ;; Remove those old items from cache which have been removed from
        ;; the feed
        (newsticker--cache-replace-age newsticker--cache
                                       name-symbol 'obsolete-old 'deleteme)
        (newsticker--cache-remove newsticker--cache name-symbol
                                  'deleteme)
        ;; Remove those new items from cache which have been removed from
        ;; the feed.  Or keep them as `obsolete'
        (if (not newsticker-keep-obsolete-items)
            (newsticker--cache-remove newsticker--cache
                                      name-symbol 'obsolete-new)
          (setq newsticker--cache
                (newsticker--cache-mark-expired
                 newsticker--cache name-symbol 'obsolete 'obsolete-expired
                 newsticker-obsolete-item-max-age))
          (newsticker--cache-remove newsticker--cache
                                    name-symbol 'obsolete-expired)
          (newsticker--cache-replace-age newsticker--cache
                                         name-symbol 'obsolete-new
                                         'obsolete))
        (newsticker--update-process-ids)
        ;; setup scrollable text
        (when (= 0 (length newsticker--process-ids))
          (when (fboundp 'newsticker--ticker-text-setup) ;silence
                                                         ;compiler
                                                         ;warnings
            (newsticker--ticker-text-setup)))
        (setq newsticker--latest-update-time (current-time))
        (when something-was-added
          ;; FIXME: should we care about removed items as well?
Ulf Jasper's avatar
Ulf Jasper committed
1032 1033
          (newsticker--cache-save-feed
           (newsticker--cache-get-feed name-symbol))
Ulf Jasper's avatar
Ulf Jasper committed
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
          (when (fboundp 'newsticker--buffer-set-uptodate) ;silence
                                                           ;compiler
                                                           ;warnings
            (newsticker--buffer-set-uptodate nil)))
        ;; kill the process buffer if wanted
        (unless newsticker-debug
          (kill-buffer buffer))
        ;; launch retrieval of image
        (when (and imageurl newsticker--download-logos)
          (newsticker--image-get name imageurl)))))
  (when newsticker--sentinel-callback
    (funcall newsticker--sentinel-callback)))

(defun newsticker--get-logo-url-atom-1.0 (node)
  "Return logo URL from atom 1.0 data in NODE."
  (car (xml-node-children
        (car (xml-get-children node 'logo)))))

(defun newsticker--get-logo-url-atom-0.3 (node)
  "Return logo URL from atom 0.3 data in NODE."
  (car (xml-node-children
        (car (xml-get-children (car (xml-get-children node 'image)) 'url)))))

(defun newsticker--get-logo-url-rss-2.0 (node)
  "Return logo URL from RSS 2.0 data in NODE."
  (car (xml-node-children
        (car (xml-get-children
              (car (xml-get-children
                    (car (xml-get-children node 'channel)) 'image)) 'url)))))

(defun newsticker--get-logo-url-rss-1.0 (node)
  "Return logo URL from RSS 1.0 data in NODE."
  (car (xml-node-children
        (car (xml-get-children (car (xml-get-children node 'image)) 'url)))))

(defun newsticker--get-logo-url-rss-0.92 (node)
  "Return logo URL from RSS 0.92 data in NODE."
  (car (xml-node-children
        (car (xml-get-children (car (xml-get-children node 'image)) 'url)))))

(defun newsticker--get-logo-url-rss-0.91 (node)
  "Return logo URL from RSS 0.91 data in NODE."
  (car (xml-node-children
        (car (xml-get-children (car (xml-get-children node 'image)) 'url)))))

(defun newsticker--parse-atom-0.3 (name time topnode)
  "Parse Atom 0.3 data.
Return value as well as arguments NAME, TIME, and TOPNODE are the
same as in `newsticker--parse-atom-1.0'."
  (newsticker--debug-msg "Parsing Atom 0.3 feed %s" name)
  (let (new-feed new-item)
    (setq new-feed (newsticker--parse-generic-feed
                    name time
                    ;; title
                    (car (xml-node-children
                          (car (xml-get-children topnode 'title))))
                    ;; desc
                    (car (xml-node-children
                          (car (xml-get-children topnode 'content))))
                    ;; link
                    (xml-get-attribute
                     (car (xml-get-children topnode 'link)) 'href)
                    ;; extra-elements
                    (xml-node-children topnode)))
    (setq new-item (newsticker--parse-generic-items
                    name time (xml-get-children topnode 'entry)
                    ;; title-fn
                    (lambda (node)
                      (car (xml-node-children
                            (car (xml-get-children node 'title)))))
                    ;; desc-fn
                    (lambda (node)
                      (or (car (xml-node-children
                                (car (xml-get-children node 'content))))
                          (car (xml-node-children
                                (car (xml-get-children node 'summary))))))
                    ;; link-fn
                    (lambda (node)
                      (xml-get-attribute
                       (car (xml-get-children node 'link)) 'href))
                    ;; time-fn
                    (lambda (node)
                      (newsticker--decode-rfc822-date
                            (car (xml-node-children
                                  (car (xml-get-children node 'modified))))))
                    ;; guid-fn
                    (lambda (node)
                      (newsticker--guid-to-string
                       (assoc 'guid (xml-node-children node))))
                    ;; extra-fn
                    (lambda (node)
                      (xml-node-children node))))
    (or new-item new-feed)))

(defun newsticker--parse-atom-1.0 (name time topnode)
  "Parse Atom 1.0 data.
Argument NAME gives the name of a news feed.  TIME gives the
system time at which the data have been retrieved.  TOPNODE
contains the feed data as returned by the xml parser.

For the Atom 1.0 specification see
1135
URL `http://www.atompub.org/2005/08/17/draft-ietf-atompub-format-11.html'"
Ulf Jasper's avatar
Ulf Jasper committed
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
  (newsticker--debug-msg "Parsing Atom 1.0 feed %s" name)
  (let (new-feed new-item)
    (setq new-feed (newsticker--parse-generic-feed
                    name time
                    ;; title
                    (car (xml-node-children
                          (car (xml-get-children topnode 'title))))
                    ;; desc
                    (car (xml-node-children
                          (car (xml-get-children topnode 'subtitle))))
                    ;; link
                    (lambda (node)
                      (xml-get-attribute
                       (car (xml-get-children node 'link)) 'href))
                    ;; extra-elements
                    (xml-node-children topnode)))
    (setq new-item (newsticker--parse-generic-items
                    name time (xml-get-children topnode 'entry)
                    ;; title-fn
                    (lambda (node)
                      (car (xml-node-children
                            (car (xml-get-children node 'title)))))
                    ;; desc-fn
                    (lambda (node)
                      (or (car (xml-node-children
                                (car (xml-get-children node 'content))))
                          (car (xml-node-children
                                (car (xml-get-children node 'summary))))))
                    ;; link-fn
                    (lambda (node)
                      (xml-get-attribute
                       (car (xml-get-children node 'link)) 'href))
                    ;; time-fn
                    (lambda (node)
                      (newsticker--decode-iso8601-date
                       (or (car (xml-node-children
                                 (car (xml-get-children node 'updated))))
                           (car (xml-node-children
                                 (car (xml-get-children node 'published)))))))
                    ;; guid-fn
                    (lambda (node)
                      (car (xml-node-children
                            (car (xml-get-children node 'id)))))
                    ;; extra-fn
                    (lambda (node)
                      (xml-node-children node))))
    (or new-item new-feed)))

(defun newsticker--parse-rss-0.91 (name time topnode)
  "Parse RSS 0.91 data.
Return value as well as arguments NAME, TIME, and TOPNODE are the
same as in `newsticker--parse-atom-1.0'.

1189 1190
For the RSS 0.91 specification see URL `http://backend.userland.com/rss091'
or URL `http://my.netscape.com/publish/formats/rss-spec-0.91.html'."
Ulf Jasper's avatar
Ulf Jasper committed
1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
  (newsticker--debug-msg "Parsing RSS 0.91 feed %s" name)
  (let* ((channelnode (car (xml-get-children topnode 'channel)))
         (pub-date (newsticker--decode-rfc822-date
                    (car (xml-node-children
                          (car (xml-get-children channelnode 'pubDate))))))
         is-new-feed has-new-items)
    (setq is-new-feed (newsticker--parse-generic-feed
                       name time
                       ;; title
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'title))))
                       ;; desc
                       (car (xml-node-children
                             (car (xml-get-children channelnode
                                                    'description))))
                       ;; link
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'link))))
                       ;; extra-elements
                       (xml-node-children channelnode)))
    (setq has-new-items (newsticker--parse-generic-items
                         name time (xml-get-children channelnode 'item)
                         ;; title-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'title)))))
                         ;; desc-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'description)))))
                         ;; link-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'link)))))
                         ;; time-fn
                         (lambda (node)
                           (newsticker--decode-rfc822-date
                            (car (xml-node-children
                                  (car (xml-get-children node 'pubDate))))))
                         ;; guid-fn
                         (lambda (node)
                           nil)
                         ;; extra-fn
                         (lambda (node)
                           (xml-node-children node))))
    (or has-new-items is-new-feed)))

(defun newsticker--parse-rss-0.92 (name time topnode)
  "Parse RSS 0.92 data.
Return value as well as arguments NAME, TIME, and TOPNODE are the
same as in `newsticker--parse-atom-1.0'.

1243
For the RSS 0.92 specification see URL `http://backend.userland.com/rss092'."
Ulf Jasper's avatar
Ulf Jasper committed
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
  (newsticker--debug-msg "Parsing RSS 0.92 feed %s" name)
  (let* ((channelnode (car (xml-get-children topnode 'channel)))
         (pub-date (newsticker--decode-rfc822-date
                    (car (xml-node-children
                          (car (xml-get-children channelnode 'pubDate))))))
         is-new-feed has-new-items)
    (setq is-new-feed (newsticker--parse-generic-feed
                       name time
                       ;; title
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'title))))
                       ;; desc
                       (car (xml-node-children
                             (car (xml-get-children channelnode
                                                    'description))))
                       ;; link
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'link))))
                       ;; extra-elements
                       (xml-node-children channelnode)))
    (setq has-new-items (newsticker--parse-generic-items
                         name time (xml-get-children channelnode 'item)
                         ;; title-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'title)))))
                         ;; desc-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'description)))))
                         ;; link-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'link)))))
                         ;; time-fn
                         (lambda (node)
                           (newsticker--decode-rfc822-date
                            (car (xml-node-children
                                  (car (xml-get-children node 'pubDate))))))
                         ;; guid-fn
                         (lambda (node)
                           nil)
                         ;; extra-fn
                         (lambda (node)
                           (xml-node-children node))))
    (or has-new-items is-new-feed)))

(defun newsticker--parse-rss-1.0 (name time topnode)
  "Parse RSS 1.0 data.
Return value as well as arguments NAME, TIME, and TOPNODE are the
same as in `newsticker--parse-atom-1.0'.

1296
For the RSS 1.0 specification see URL `http://web.resource.org/rss/1.0/spec'."
Ulf Jasper's avatar
Ulf Jasper committed
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
  (newsticker--debug-msg "Parsing RSS 1.0 feed %s" name)
  (let* ((channelnode (car (xml-get-children topnode 'channel)))
         is-new-feed has-new-items)
    (setq is-new-feed (newsticker--parse-generic-feed
                       name time
                       ;; title
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'title))))
                       ;; desc
                       (car (xml-node-children
                             (car (xml-get-children channelnode
                                                    'description))))
                       ;; link
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'link))))
                       ;; extra-elements
                       (xml-node-children channelnode)))
    (setq has-new-items (newsticker--parse-generic-items
                         name time (xml-get-children topnode 'item)
                         ;; title-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'title)))))
                         ;; desc-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node
                                                        'description)))))
                         ;; link-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'link)))))
                         ;; time-fn
                         (lambda (node)
                           (newsticker--decode-iso8601-date
                            (car (xml-node-children
                                  (car (xml-get-children node 'dc:date))))))
                         ;; guid-fn
                         (lambda (node)
                           nil)
                         ;; extra-fn
                         (lambda (node)
                           (xml-node-children node))))
    (or has-new-items is-new-feed)))

(defun newsticker--parse-rss-2.0 (name time topnode)
  "Parse RSS 2.0 data.
Return value as well as arguments NAME, TIME, and TOPNODE are the
same as in `newsticker--parse-atom-1.0'.

1347
For the RSS 2.0 specification see URL `http://blogs.law.harvard.edu/tech/rss'."
Ulf Jasper's avatar
Ulf Jasper committed
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410
  (newsticker--debug-msg "Parsing RSS 2.0 feed %s" name)
  (let* ((channelnode (car (xml-get-children topnode 'channel)))
         is-new-feed has-new-items)
    (setq is-new-feed (newsticker--parse-generic-feed
                       name time
                       ;; title
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'title))))
                       ;; desc
                       (car (xml-node-children
                             (car (xml-get-children channelnode
                                                    'description))))
                       ;; link
                       (car (xml-node-children
                             (car (xml-get-children channelnode 'link))))
                       ;; extra-elements
                       (xml-node-children channelnode)))
    (setq has-new-items (newsticker--parse-generic-items
                         name time (xml-get-children channelnode 'item)
                         ;; title-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'title)))))
                         ;; desc-fn
                         (lambda (node)
                           (or (car (xml-node-children
                                     (car (xml-get-children node
                                                            'content:encoded))))
                               (car (xml-node-children
                                     (car (xml-get-children node
                                                            'description))))))
                         ;; link-fn
                         (lambda (node)
                           (car (xml-node-children
                                 (car (xml-get-children node 'link)))))
                         ;; time-fn
                         (lambda (node)
                           (newsticker--decode-rfc822-date
                            (car (xml-node-children
                                  (car (xml-get-children node 'pubDate))))))
                         ;; guid-fn
                         (lambda (node)
                           (newsticker--guid-to-string
                            (assoc 'guid (xml-node-children node))))
                         ;; extra-fn
                         (lambda (node)
                           (xml-node-children node))))
    (or has-new-items is-new-feed)))

(defun newsticker--parse-generic-feed (name time title desc link
                                            extra-elements)
  "Parse generic news feed data.
Argument NAME gives the name of a news feed.  TIME gives the
system time at which the data have been retrieved.

The arguments TITLE, DESC, LINK, and EXTRA-ELEMENTS give the feed's title,
description, link, and extra elements resp."
  (let ((title (or title "[untitled]"))
        (link (or link ""))
        (old-item nil)
        (position 0)
        (something-was-added nil))
    ;; decode numeric entities
Ulf Jasper's avatar
Ulf Jasper committed
1411 1412 1413
    (setq title (xml-substitute-numeric-entities title))
    (setq desc  (xml-substitute-numeric-entities desc))
    (setq link  (xml-substitute-numeric-entities link))
Ulf Jasper's avatar
Ulf Jasper committed
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
    ;; remove whitespace from title, desc, and link
    (setq title (newsticker--remove-whitespace title))
    (setq desc (newsticker--remove-whitespace desc))
    (setq link (newsticker--remove-whitespace link))

    ;; handle the feed itself
    (unless (newsticker--cache-contains newsticker--cache
                                        (intern name) title
                                        desc link 'feed)
      (setq something-was-added t))
    (setq newsticker--cache
          (newsticker--cache-add newsticker--cache (intern name)
                                 title desc link time 'feed position
                                 extra-elements time 'feed))
    something-was-added))

(defun newsticker--parse-generic-items (name time itemlist
                                             title-fn desc-fn
                                             link-fn time-fn
                                             guid-fn extra-fn)
  "Parse generic news feed data.
Argument NAME gives the name of a news feed.  TIME gives the
system time at which the data have been retrieved.  ITEMLIST
contains the news items returned by the xml parser.

The arguments TITLE-FN, DESC-FN, LINK-FN, TIME-FN, GUID-FN, and
EXTRA-FN give functions for extracting title, description, link,
time, guid, and extra-elements resp.  They are called with one
argument, which is one of the items in ITEMLIST."
  (let (title desc link
        (old-item nil)
        (position 0)
        (something-was-added nil))
    ;; gather all items for this feed
    (mapc (lambda (node)
            (setq position (1+ position))
            (setq title (or (funcall title-fn node) "[untitled]"))
            (setq desc (funcall desc-fn node))
            (setq link (or (funcall link-fn node) ""))
            (setq time (or (funcall time-fn node) time))
            ;; It happened that the title or description
            ;; contained evil HTML code that confused the
            ;; xml parser.  Therefore:
            (unless (stringp title)
              (setq title (prin1-to-string title)))
            (unless (or (stringp desc) (not desc))
              (setq desc (prin1-to-string desc)))
            ;; ignore items with empty title AND empty desc
            (when (or (> (length title) 0)
                      (> (length desc) 0))
              ;; decode numeric entities
Ulf Jasper's avatar
Ulf Jasper committed
1465
              (setq title (xml-substitute-numeric-entities title))
Ulf Jasper's avatar
Ulf Jasper committed
1466
              (when desc
Ulf Jasper's avatar
Ulf Jasper committed
1467 1468
                (setq desc  (xml-substitute-numeric-entities desc)))
              (setq link (xml-substitute-numeric-entities link))
Ulf Jasper's avatar
Ulf Jasper committed
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
              ;; remove whitespace from title, desc, and link
              (setq title (newsticker--remove-whitespace title))
              (setq desc (newsticker--remove-whitespace desc))
              (setq link (newsticker--remove-whitespace link))
              ;; add data to cache
              ;; do we have this item already?
              (let* ((guid (funcall guid-fn node)))
                ;;(message "guid=%s" guid)
                (setq old-item
                      (newsticker--cache-contains newsticker--cache
                                                  (intern name) title
                                                  desc link nil guid)))
              ;; add this item, or mark it as old, or do nothing
              (let ((age1 'new)
                    (age2 'old)
                    (item-new-p nil))
                (if old-item
                    (let ((prev-age (newsticker--age old-item)))
                      (unless newsticker-automatically-mark-items-as-old
                        ;; Some feeds deliver items multiply, the
                        ;; first time we find an 'obsolete-old one the
                        ;; cache, the following times we find an 'old
                        ;; one
                        (if (memq prev-age '(obsolete-old old))
                            (setq age2 'old)
                          (setq age2 'new)))
                      (if (eq prev-age 'immortal)
                          (setq age2 'immortal))
                      (setq time (newsticker--time old-item)))
                  ;; item was not there
                  (setq item-new-p t)
                  (setq something-was-added t))
                (setq newsticker--cache
                      (newsticker--cache-add
                       newsticker--cache (intern name) title desc link
                       time age1 position (funcall extra-fn node)
                       time age2))
                (when item-new-p
                  (let ((item (newsticker--cache-contains
                               newsticker--cache (intern name) title
                               desc link nil)))
                    (if newsticker-auto-mark-filter-list
                        (newsticker--run-auto-mark-filter name item))
                    (run-hook-with-args
                     'newsticker-new-item-functions name item))))))
          itemlist)
    something-was-added))

;; ======================================================================
;;; Misc
;; ======================================================================

(defun newsticker--remove-whitespace (string)
  "Remove leading and trailing whitespace from STRING."
  ;; we must have ...+ but not ...* in the regexps otherwise xemacs loops
  ;; endlessly...
  (when (and string (stringp string))
    (replace-regexp-in-string
     "[ \t\r\n]+$" ""
     (replace-regexp-in-string "^[ \t\r\n]+" "" string))))

(defun newsticker--do-forget-preformatted (item)
  "Forget pre-formatted data for ITEM.
Remove the pre-formatted from `newsticker--cache'."
  (if (nthcdr 7 item)
      (setcar (nthcdr 7 item) nil))
  (if (nthcdr 6 item)
      (setcar (nthcdr 6 item) nil)))

(defun newsticker--forget-preformatted ()
  "Forget all cached pre-formatted data.
Remove the pre-formatted from `newsticker--cache'."
  (mapc (lambda (feed)
          (mapc 'newsticker--do-forget-preformatted
                (cdr feed)))
        newsticker--cache)
  (when (fboundp 'newsticker--buffer-set-uptodate)
    (newsticker--buffer-set-uptodate nil)))

(defun newsticker--debug-msg (string &rest args)
  "Print newsticker debug messages.
This function calls `message' with arguments STRING and ARGS, if
`newsticker-debug' is non-nil."
  (and newsticker-debug
       ;;(not (active-minibuffer-window))
       ;;(not (current-message))
       (apply 'message string args)))

(defun newsticker--decode-iso8601-date (iso8601-string)
  "Return ISO8601-STRING in format like `decode-time'.
Converts from ISO-8601 to Emacs representation.
Examples:
2004-09-17T05:09:49.001+00:00
2004-09-17T05:09:49+00:00
2004-09-17T05:09+00:00
2004-09-17T05:09:49
2004-09-17T05:09
2004-09-17
2004-09
2004"
  (if iso8601-string
      (when (string-match
             (concat
              "^ *\\([0-9]\\{4\\}\\)"   ;year
              "\\(-\\([0-9]\\{2\\}\\)"  ;month
              "\\(-\\([0-9]\\{2\\}\\)"  ;day
              "\\(T"
              "\\([0-9]\\{2\\}\\):\\([0-9]\\{2\\}\\)" ;hour:minute
              "\\(:\\([0-9]\\{2\\}\\)\\(\\.[0-9]+\\)?\\)?" ;second
              ;timezone
              "\\(\\([-+Z]\\)\\(\\([0-9]\\{2\\}\\):\\([0-9]\\{2\\}\\)\\)?\\)?"
              "\\)?\\)?\\)? *$")
             iso8601-string)
        (let ((year (read (match-string 1 iso8601-string)))
              (month (read (or (match-string 3 iso8601-string)
                               "1")))
              (day (read (or (match-string 5 iso8601-string)
                             "1")))
              (hour (read (or (match-string 7 iso8601-string)
                              "0")))
              (minute (read (or (match-string 8 iso8601-string)
                                "0")))
              (second (read (or (match-string 10 iso8601-string)
                                "0")))
              (sign (match-string 13 iso8601-string))
              (offset-hour (read (or (match-string 15 iso8601-string)
                                     "0")))
              (offset-minute (read (or (match-string 16 iso8601-string)
                                       "0"))))
          (cond ((string= sign "+")
                 (setq hour (- hour offset-hour))
                 (setq minute (- minute offset-minute)))
                ((string= sign "-")
                 (setq hour (+ hour offset-hour))
                 (setq minute (+ minute offset-minute))))
          ;; if UTC subtract current-time-zone offset
          ;;(setq second (+ (car (current-time-zone)) second)))

          (condition-case nil
              (encode-time second minute hour day month year t)
            (error
             (message "Cannot decode \"%s\"" iso8601-string)
             nil))))
    nil))

(defun newsticker--decode-rfc822-date (rfc822-string)
  "Return RFC822-STRING in format like `decode-time'.
Converts from RFC822 to Emacs representation.
Examples:
Sat, 07 September 2002 00:00:01 +0100
Sat, 07 September 2002 00:00:01 MET
Sat, 07 Sep 2002 00:00:01 GMT
07 Sep 2002 00:00:01 GMT
07 Sep 2002"
  (if (and rfc822-string (stringp rfc822-string))
      (when (string-match
             (concat
              "\\s-*"
              ;; week day
              "\\(\\(Mon\\|Tue\\|Wed\\|Thu\\|Fri\\|Sat\\|Sun\\)\\s-*,?\\)?\\s-*"
              ;; day
              "\\([0-9]\\{1,2\\}\\)\\s-+"
              ;; month
              "\\(Jan\\|Feb\\|Mar\\|Apr\\|May\\|Jun\\|"
              "Jul\\|Aug\\|Sep\\|Oct\\|Nov\\|Dec\\).*?\\s-+"
              ;; year
              "\\([0-9]\\{2,4\\}\\)"
              ;; time may be missing
              "\\(\\s-+"
              ;; hour
              "\\([0-9]\\{2\\}\\)"
              ;; minute
              ":\\([0-9]\\{2\\}\\)"
              ;; second
              "\\(:\\([0-9]\\{2\\}\\)\\)?"
              ;; zone -- fixme
              "\\(\\s-+\\("
              "UT\\|GMT\\|EST\\|EDT\\|CST\\|CDT\\|MST\\|MDT\\|PST\\|PDT"
              "\\|\\([-+]\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)"
              "\\)\\)?"
              "\\)?")
             rfc822-string)
        (let ((day (read (match-string 3 rfc822-string)))
              (month-name (match-string 4 rfc822-string))
              (month 0)
              (year (read (match-string 5 rfc822-string)))
              (hour (read (or (match-string 7 rfc822-string) "0")))
              (minute (read (or (match-string 8 rfc822-string) "0")))
              (second (read (or (match-string 10 rfc822-string) "0")))
              (zone (match-string 12 rfc822-string))
              (sign (match-string 13 rfc822-string))
              (offset-hour (read (or (match-string 14 rfc822-string)
                                     "0")))
              (offset-minute (read (or (match-string 15 rfc822-string)
                                       "0")))
              ;;FIXME
              )
          (when zone
            (cond ((string= sign "+")
                   (setq hour (- hour offset-hour))
                   (setq minute (- minute offset-minute)))
                  ((string= sign "-")
                   (setq hour (+ hour offset-hour))
                   (setq minute (+ minute offset-minute)))))
          (condition-case error-data
              (let ((i 1))
                (mapc (lambda (m)
                        (if (string= month-name m)
                            (setq month i))
                        (setq i (1+ i)))
                      '("Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug"
                        "Sep" "Oct" "Nov" "Dec"))
                (encode-time second minute hour day month year t))
            (error
             (message "Cannot decode \"%s\": %s %s" rfc822-string
                      (car error-data) (cdr error-data))
             nil))))
      nil))

(defun newsticker--lists-intersect-p (list1 list2)
  "Return t if LIST1 and LIST2 share elements."
  (let ((result nil))
    (mapc (lambda (elt)
            (if (memq elt list2)
                (setq result t)))
          list1)
    result))

(defun newsticker--update-process-ids ()
  "Update list of ids of active newsticker processes.
Checks list of active processes against list of newsticker processes."
  (let ((active-procs (process-list))
        (new-list nil))
    (mapc (lambda (proc)
            (let ((id (process-id proc)))
              (if (memq id newsticker--process-ids)
                  (setq new-list (cons id new-list)))))
          active-procs)
    (setq newsticker--process-ids new-list))
  (force-mode-line-update))

;; ======================================================================
;;; Images
;; ======================================================================
Ulf Jasper's avatar
Ulf Jasper committed
1713 1714 1715 1716
(defun newsticker--images-dir ()
  "Return directory where feed images are saved."
  (concat newsticker-dir "/images"))

Ulf Jasper's avatar
Ulf Jasper committed
1717 1718 1719
(defun newsticker--image-get (feed-name url)
  "Get image of the news site FEED-NAME from URL.
If the image has been downloaded in the last 24h do nothing."
Ulf Jasper's avatar
Ulf Jasper committed
1720
  (let ((image-name (concat (newsticker--images-dir) feed-name)))
Ulf Jasper's avatar
Ulf Jasper committed
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
    (if (and (file-exists-p image-name)
             (time-less-p (current-time)
                          (time-add (nth 5 (file-attributes image-name))
                                    (seconds-to-time 86400))))
        (newsticker--debug-msg "%s: Getting image for %s skipped"
                               (format-time-string "%A, %H:%M" (current-time))
                               feed-name)
      ;; download
      (newsticker--debug-msg "%s: Getting image for %s"
                             (format-time-string "%A, %H:%M" (current-time))
                             feed-name)
      (let* ((buffername (concat " *newsticker-wget-image-" feed-name "*"))
             (item (or (assoc feed-name newsticker-url-list)
                       (assoc feed-name newsticker-url-list-defaults)
                       (error
                        "Cannot get news for %s: Check newsticker-url-list"
                        feed-name)))
             (wget-arguments (or (car (cdr (cdr (cdr (cdr item)))))
                                 newsticker-wget-arguments)))
1740
        (with-current-buffer (get-buffer-create buffername)
Ulf Jasper's avatar
Ulf Jasper committed
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
          (erase-buffer)
          ;; throw an error if there is an old wget-process around
          (if (get-process feed-name)
              (error "Another wget-process is running for image %s"
                     feed-name))
          ;; start wget
          (let* ((args (append wget-arguments (list url)))
                 (proc (apply 'start-process feed-name buffername
                              newsticker-wget-name args)))
            (set-process-coding-system proc 'no-conversion 'no-conversion)
            (set-process-sentinel proc 'newsticker--image-sentinel)))))))

(defun newsticker--image-sentinel (process event)
  "Sentinel for image-retrieving PROCESS caused by EVENT."
  (let* ((p-status (process-status process))
         (exit-status (process-exit-status process))
         (feed-name (process-name process)))
    ;; catch known errors (zombie processes, rubbish-xml, etc.)
    ;; if an error occurs the news feed is not updated!
    (catch 'oops
      (unless (and (eq p-status 'exit)
                   (= exit-status 0))
        (message "%s: Error while retrieving image from %s"
                 (format-time-string "%A, %H:%M" (current-time))
                 feed-name)
        (throw 'oops nil))
      (let (image-name)
1768
        (with-current-buffer (process-buffer process)
Ulf Jasper's avatar
Ulf Jasper committed
1769
          (setq image-name (concat (newsticker--images-dir) feed-name))
Ulf Jasper's avatar
Ulf Jasper committed
1770 1771
          (set-buffer-file-coding-system 'no-conversion)
          ;; make sure the cache dir exists
Ulf Jasper's avatar
Ulf Jasper committed
1772 1773
          (unless (file-directory-p (newsticker--images-dir))
            (make-directory (newsticker--images-dir)))
Ulf Jasper's avatar
Ulf Jasper committed
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913
          ;; write and close buffer
          (let ((require-final-newline nil)
                (backup-inhibited t)
                (coding-system-for-write 'no-conversion))
            (write-region nil nil image-name nil 'quiet))
          (set-buffer-modified-p nil)
          (kill-buffer (current-buffer)))))))

(defun newsticker--insert-image (img string)
  "Insert IMG with STRING at point."
  (insert-image img string))

;; ======================================================================
;;; HTML rendering
;; ======================================================================
(defun newsticker-htmlr-render (pos1 pos2) ;
  "Replacement for `htmlr-render'.
Renders the HTML code in the region POS1 to POS2 using htmlr."
  (let ((str (buffer-substring-no-properties pos1 pos2)))
    (delete-region pos1 pos2)
    (insert
     (with-temp-buffer
       (insert str)
       (goto-char (point-min))
       ;; begin original htmlr-render
       (when (fboundp 'htmlr-reset) (htmlr-reset))
       ;; something omitted here...
       (when (fboundp 'htmlr-step)
         (while (< (point) (point-max))
           (htmlr-step)))
       ;; end original htmlr-render
       (newsticker--remove-whitespace (buffer-string))))))

;; ======================================================================
;;; Manipulation of cached data
;; ======================================================================
(defun newsticker--cache-set-preformatted-contents (item contents)
  "Set preformatted contents of ITEM to CONTENTS."
  (if (nthcdr 6 item)
      (setcar (nthcdr 6 item) contents)
    (setcdr (nthcdr 5 item) (list contents))))

(defun newsticker--cache-set-preformatted-title (item title)
  "Set preformatted title of ITEM to TITLE."
  (if (nthcdr 7 item)
      (setcar (nthcdr 7 item) title)
    (setcdr (nthcdr 6 item) title)))

(defun newsticker--cache-replace-age (data feed old-age new-age)
  "Mark all items in DATA in FEED which carry age OLD-AGE with NEW-AGE.
If FEED is 'any it applies to all feeds.  If OLD-AGE is 'any,
all marks are replaced by NEW-AGE.  Removes all pre-formatted contents."
  (mapc (lambda (a-feed)
          (when (or (eq feed 'any)
                    (eq (car a-feed) feed))
            (let ((items (cdr a-feed)))
              (mapc (lambda (item)
                      (when (or (eq old-age 'any)
                                (eq (newsticker--age item) old-age))
                        (setcar (nthcdr 4 item) new-age)
                        (newsticker--do-forget-preformatted item)))
                    items))))
        data)
  data)

(defun newsticker--cache-mark-expired (data feed old-age new-age time)
  "Mark all expired entries.
This function sets the age entries in DATA in the feed FEED.  If
an item's age is OLD-AGE it is set to NEW-AGE if the item is
older than TIME."
  (mapc
   (lambda (a-feed)
     (when (or (eq feed 'any)
               (eq (car a-feed) feed))
       (let ((items (cdr a-feed)))
         (mapc
          (lambda (item)
            (when (eq (newsticker--age item) old-age)
              (let ((exp-time (time-add (newsticker--time item)
                                        (seconds-to-time time))))
                (when (time-less-p exp-time (current-time))
                  (newsticker--debug-msg
                   "Item `%s' from %s has expired on %s"
                   (newsticker--title item)
                   (format-time-string "%Y-%02m-%d, %H:%M"
                                       (newsticker--time item))
                   (format-time-string "%Y-%02m-%d, %H:%M" exp-time))
                  (setcar (nthcdr 4 item) new-age)))))
          items))))
   data)
  data)

(defun newsticker--cache-contains (data feed title desc link age
                                        &optional guid)
  "Check DATA whether FEED contains an item with the given properties.
This function returns the contained item or nil if it is not
contained.
The properties which are checked are TITLE, DESC, LINK, AGE, and
GUID.  In general all properties must match in order to return a
certain item, except for the following cases.

If AGE equals 'feed the TITLE, DESCription and LINK do not
matter.  If DESC is nil it is ignored as well.  If
`newsticker-desc-comp-max' is non-nil, only the first
`newsticker-desc-comp-max' characters of DESC are taken into
account.

If GUID is non-nil it is sufficient to match this value, and the
other properties are ignored."
  ;;(newsticker--debug-msg "Looking for %s guid=%s" title  guid)
  (condition-case nil
      (catch 'found
        (when (and desc newsticker-desc-comp-max
                   (> (length desc) newsticker-desc-comp-max))
          (setq desc (substring desc 0 newsticker-desc-comp-max)))
        (mapc
         (lambda (this-feed)
           (when (eq (car this-feed) feed)
             (mapc (lambda (anitem)
                     (when (cond (guid
                                  ;; global unique id can match
                                  (string= guid (newsticker--guid anitem)))
                                 (t;;FIXME?
                                  (or
                                   ;; or title, desc, etc.
                                   (and
                                    ;;(or (not (eq age 'feed))
                                    ;;  (eq (newsticker--age anitem) 'feed))
                                    (string= (newsticker--title anitem)
                                             title)
                                    (or (not link)
                                        (string= (newsticker--link anitem)
                                                 link))
                                    (or (not desc)
                                        (if (and desc newsticker-desc-comp-max
                                                 (> (length (newsticker--desc
                                                             anitem))
                                                    newsticker-desc-comp-max))
                                            (string= (substring
                                                      (newsticker--desc anitem)
Ulf Jasper's avatar
Ulf Jasper committed
1914
                                                      0
Ulf Jasper's avatar
Ulf Jasper committed
1915 1916 1917 1918
                                                      newsticker-desc-comp-max)
                                                     desc)
                                          (string= (newsticker--desc anitem)
                                                   desc)))))))
Ulf Jasper's avatar
Ulf Jasper committed
1919
                       ;;(newsticker--debug-msg "Found %s guid=%s"
Ulf Jasper's avatar
Ulf Jasper committed
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947
                       ;; (newsticker--title anitem)
                       ;;                     (newsticker--guid anitem))
                       (throw 'found anitem)))
                   (cdr this-feed))))
         data)
        ;;(newsticker--debug-msg "Found nothing")
        nil)
    (error nil)))

(defun newsticker--cache-add (data feed-name-symbol title desc link time age
                                   position extra-elements
                                   &optional updated-time updated-age
                                   preformatted-contents
                                   preformatted-title)
  "Add another item to cache data.
Add to DATA in the FEED-NAME-SYMBOL an item with TITLE, DESC,
LINK, TIME, AGE, POSITION, and EXTRA-ELEMENTS.  If this item is
contained already, its time is set to UPDATED-TIME, its mark is
set to UPDATED-AGE, and its pre-formatted contents is set to
PREFORMATTED-CONTENTS and PREFORMATTED-TITLE.  Returns the age
which the item got."
  (let* ((guid (newsticker--guid-to-string (assoc 'guid extra-elements)))
         (item (newsticker--cache-contains data feed-name-symbol title desc link
                                           age guid)))
    ;;(message "guid=%s" guid)
    (if item
        ;; does exist already -- change age, update time and position
        (progn
Ulf Jasper's avatar
Ulf Jasper committed
1948
          ;;(newsticker--debug-msg "Updating item %s %s %s %s %s -> %s %s
Ulf Jasper's avatar
Ulf Jasper committed
1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071
          ;;                   (guid %s -> %s)"
          ;;                   feed-name-symbol title link time age
          ;;                 updated-time updated-age
          ;;               guid (newsticker--guid item))
          (if (nthcdr 5 item)
              (setcar (nthcdr 5 item) position)
            (setcdr (nthcdr 4 item) (list position)))
          (setcar (nthcdr 4 item) updated-age)
          (if updated-time
              (setcar (nthcdr 3 item) updated-time))
          ;; replace cached pre-formatted contents
	  (newsticker--cache-set-preformatted-contents
	   item preformatted-contents)
	  (newsticker--cache-set-preformatted-title
	   item preformatted-title))
      ;; did not exist or age equals 'feed-name-symbol
      (setq item (list title desc link time age position preformatted-contents
                       preformatted-title extra-elements))
      ;;(newsticker--debug-msg "Adding item %s" item)
      (catch 'found
        (mapc (lambda (this-feed)
                (when (eq (car this-feed) feed-name-symbol)
                  (setcdr this-feed (nconc (cdr this-feed) (list item)))
                  (throw 'found this-feed)))
              data)
        ;; the feed is not contained
        (add-to-list 'data (list feed-name-symbol item) t))))
  data)

(defun newsticker--cache-remove (data feed-symbol age)
  "Remove all entries from DATA in the feed FEED-SYMBOL with AGE.
FEED-SYMBOL may be 'any.  Entries from old feeds, which are no longer in
`newsticker-url-list' or `newsticker-url-list-defaults', are removed as
well."
  (let* ((pos data)
         (feed (car pos))
         (last-pos nil))
    (while feed
      (if (or (assoc (symbol-name (car feed)) newsticker-url-list)
              (assoc (symbol-name (car feed)) newsticker-url-list-defaults))
          ;; feed is still valid=active
          ;; (message "Keeping feed %s" (car feed))
          (if  (or (eq feed-symbol 'any)
                   (eq feed-symbol (car feed)))
              (let* ((item-pos (cdr feed))
                     (item (car item-pos))
                     (prev-pos nil))
                (while item
                  ;;(message "%s" (car item))
                  (if (eq age (newsticker--age item))
                      ;; remove this item
                      (progn
                        ;;(message "Removing item %s" (car item))
                        (if prev-pos
                            (setcdr prev-pos (cdr item-pos))
                          (setcdr feed (cdr item-pos))))
                    ;;(message "Keeping item %s" (car item))
                    (setq prev-pos item-pos))
                  (setq item-pos (cdr item-pos))
                  (setq item (car item-pos)))))
        ;; feed is not active anymore
        ;; (message "Removing feed %s" (car feed))
        (if last-pos
            (setcdr last-pos (cdr pos))
          (setq data (cdr pos))))
      (setq last-pos pos)
      (setq pos (cdr pos))
      (setq feed (car pos)))))

;; ======================================================================
;;; Sorting
;; ======================================================================
(defun newsticker--cache-item-compare-by-time (item1 item2)
  "Compare two news items ITEM1 and ITEM2 by comparing their time values."
  (catch 'result
    (let ((age1 (newsticker--age item1))
          (age2 (newsticker--age item2)))
      (if (not (eq age1 age2))
          (cond ((eq age1 'obsolete)
                 (throw 'result nil))
                ((eq age2 'obsolete)
                 (throw 'result t)))))
    (let* ((time1 (newsticker--time item1))
           (time2 (newsticker--time item2)))
      (cond ((< (nth 0 time1) (nth 0 time2))
             nil)
            ((> (nth 0 time1) (nth 0 time2))
             t)
            ((< (nth 1 time1) (nth 1 time2))
             nil)
            ((> (nth 1 time1) (nth 1 time2))
             t)
            ((< (or (nth 2 time1) 0) (or (nth 2 time2) 0))
             nil)
            ((> (or (nth 2 time1) 0) (or (nth 2 time2) 0))
             t)
            (t
             nil)))))

(defun newsticker--cache-item-compare-by-title (item1 item2)
  "Compare ITEM1 and ITEM2 by comparing their titles."
  (catch 'result
    (let ((age1 (newsticker--age item1))
          (age2 (newsticker--age item2)))
      (if (not (eq age1 age2))
          (cond ((eq age1 'obsolete)
                 (throw 'result nil))
                ((eq age2 'obsolete)
                 (throw 'result t)))))
    (string< (newsticker--title item1) (newsticker--title item2))))

(defun newsticker--cache-item-compare-by-position (item1 item2)
  "Compare ITEM1 and ITEM2 by comparing their original positions."
  (catch 'result
    (let ((age1 (newsticker--age item1))
          (age2 (newsticker--age item2)))
      (if (not (eq age1 age2))
          (cond ((eq age1 'obsolete)
                 (throw 'result nil))
                ((eq age2 'obsolete)
                 (throw 'result t)))))
    (< (or (newsticker--pos item1) 0) (or (newsticker--pos item2) 0))))

Ulf Jasper's avatar
Ulf Jasper committed
2072
(defun newsticker--cache-save-version1 ()
Ulf Jasper's avatar
Ulf Jasper committed
2073 2074 2075 2076 2077 2078 2079 2080
  "Update and save newsticker cache file."
  (interactive)
  (newsticker--cache-update t))

(defun newsticker--cache-update (&optional save)
  "Update newsticker cache file.
If optional argument SAVE is not nil the cache file is saved to disk."
  (save-excursion
Ulf Jasper's avatar
Ulf Jasper committed
2081 2082
    (unless (file-directory-p newsticker-dir)
      (make-directory newsticker-dir t))
2083 2084 2085 2086
    (let ((coding-system-for-write 'utf-8)
          (buf (find-file-noselect newsticker-cache-filename)))
      (when buf
        (set-buffer buf)
Ulf Jasper's avatar
Ulf Jasper committed
2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
        (setq buffer-undo-list t)
        (erase-buffer)
        (insert ";; -*- coding: utf-8 -*-\n")
        (insert (prin1-to-string newsticker--cache))
        (when save
          (save-buffer))))))

(defun newsticker--cache-get-feed (feed)
  "Return the cached data for the feed FEED.
FEED is a symbol!"
  (assoc feed newsticker--cache))

Ulf Jasper's avatar
Ulf Jasper committed
2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
(defun newsticker--cache-dir ()
  "Return directory for saving cache data."
  (concat newsticker-dir "/feeds"))

(defun newsticker--cache-save ()
    "Save cache data for all feeds."
    (unless (file-directory-p newsticker-dir)
      (make-directory newsticker-dir t))
    (mapc 'newsticker--cache-save-feed newsticker--cache)
    nil)

(defun newsticker--cache-save-feed (feed)
  "Save cache data for FEED."
  (let ((dir (concat (newsticker--cache-dir) "/" (symbol-name (car feed)))))
    (unless (file-directory-p dir)
      (make-directory dir t))
    (let ((coding-system-for-write 'utf-8))
      (with-temp-file (concat dir "/data")
        (insert ";; -*- coding: utf-8 -*-\n")
        (insert (prin1-to-string (cdr feed)))))))

(defun newsticker--cache-read-version1 ()
  "Read version1 cache data."
  (let ((coding-system-for-read 'utf-8))
    (when (file-exists-p newsticker-cache-filename)
      (with-temp-buffer
        (insert-file-contents newsticker-cache-filename)
        (goto-char (point-min))
        (condition-case nil
            (setq newsticker--cache (read (current-buffer)))
          (error
           (message "Error while reading newsticker cache file!")
           (setq newsticker--cache nil)))))))

(defun newsticker--cache-read ()
  "Read cache data."
  (setq newsticker--cache nil)
  (if (file-exists-p newsticker-cache-filename)
      (progn
        (when (y-or-n-p "Old newsticker cache file exists.  Read it? ")
          (newsticker--cache-read-version1))
Ulf Jasper's avatar
Ulf Jasper committed
2140 2141
        (when (y-or-n-p (format "Delete old newsticker cache file? "))
          (delete-file newsticker-cache-filename)))
Ulf Jasper's avatar
Ulf Jasper committed
2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
    (mapc (lambda (f)
            (newsticker--cache-read-feed (car f)))
          (append newsticker-url-list-defaults newsticker-url-list))))

(defun newsticker--cache-read-feed (feed-name)
  "Read cache data for feed named FEED-NAME."
  (let ((file-name (concat (newsticker--cache-dir) "/" feed-name "/data"))
        (coding-system-for-read 'utf-8))
    (when (file-exists-p file-name)
      (with-temp-buffer
        (insert-file-contents file-name)
        (goto-char (point-min))
        (condition-case nil
            (add-to-list 'newsticker--cache (cons (intern feed-name)
                                                  (read (current-buffer))))
          (error
           (message "Error while reading newsticker cache file %s!"
2159 2160
                    file-name)
           (setq newsticker--cache nil)))))))
Ulf Jasper's avatar
Ulf Jasper committed
2161

Ulf Jasper's avatar
Ulf Jasper committed
2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340