Author: cludwig Date: Sat Jan 31 20:15:50 2009 New Revision: 8 Log: fix import via atom feeds; make use of dependency and updated fields Modified: trunk/src/atom/atom.lisp trunk/src/rest_interface/read.lisp Modified: trunk/src/atom/atom.lisp ============================================================================== --- trunk/src/atom/atom.lisp (original) +++ trunk/src/atom/atom.lisp Sat Jan 31 20:15:50 2009 @@ -19,6 +19,7 @@ :subfeeds :tm-id :tm-id-p + :updated :*base-url* :*tm-feed*)) Modified: trunk/src/rest_interface/read.lisp ============================================================================== --- trunk/src/rest_interface/read.lisp (original) +++ trunk/src/rest_interface/read.lisp Sat Jan 31 20:15:50 2009 @@ -44,7 +44,7 @@ (read-url snapshot-feed-url))) (parse-snapshots-feed snapshot-feed))) -(defun import-fragments-feed (fragment-feed-url) +(defun import-fragments-feed (fragment-feed-url imported-snapshot-entry) ;a bit of a borderline case if that should be here or in the ;importer. Since it deals with the network interface, I think it ;makes sense to have it here, though @@ -56,8 +56,9 @@ ((top (d:get-item-by-psi (psi entry) :revision revision)) (xtm-id (atom:id entry)) (source-locator (source-locator-prefix feed))) - ;check if xtm-id has already been imported. If so, don't do it again - (unless (xtm-id-p xtm-id) + ;check if xtm-id has already been imported or if the entry is older + ;than the snapshot feed. If so, don't do it again + (unless (or (xtm-id-p xtm-id) (string> (atom:updated entry) (atom:updated imported-snapshot-entry))) (when top (mark-as-deleted top :source-locator source-locator :revision revision)) (format t "Fragment feed: ~a~&" (link entry)) @@ -73,20 +74,27 @@ (d:get-item-by-psi (psi entry) :revision revision) ;works even if the topic is only created during import :source-locator source-locator :revision revision)))))) +(defun string-max (string-list &optional (max nil)) + (cond + ((null string-list) + max) + ((string> (first string-list) max) + (string-max (rest string-list) (first string-list))) + (t + (string-max (rest string-list) max)))) + (defun import-snapshots-feed (snapshot-feed-url) - ;this would have to find the oldest (?) snapshot and import that. It - ;subsequently applies all fragments against it + ;this finds the most recent snapshot and imports that. It returns the entry + ;corresponding to that snapshot - ;sets source locator (let ((feed (read-snapshots-feed snapshot-feed-url)) (revision (get-revision))) - ;TODO: we lie for now and claim that the first entry always - ;represents the oldest snapshot we lie in addition in that we - ;import the snapshotfeed outright, not looking at the source - ;locator (let* - ((entry (first (entries feed))) + ((entry + (find + (string-max (mapcar #'atom:updated (atom:entries feed))) + (atom:entries feed) :key #'updated :test #'string=)) (xtm-id (id entry))) ;;that *should* be the algorithm... ;; If a client has a local topic map that contains topic map @@ -100,11 +108,12 @@ (importer-xtm1.0 (dom:document-element (cxml:parse-rod (read-url (link entry)) (cxml-dom:make-dom-builder))) - :xtm-id xtm-id :revision revision))))) + :xtm-id xtm-id :revision revision)) + entry))) -(defun import-tm-feed (feed-url) - "takes the feed url, imports the first snapshot if necessary and -then applies all fragments to it" +(defun import-tm-feed (feed-url &optional (processed-feed-urls nil)) + "takes the feed url of a collection feed, imports the first snapshot if +necessary and then applies all fragments to it" ;the implementation may be a bit brutal, but relies only on ;guaranteed rel-attributes on the links (let* @@ -123,10 +132,26 @@ (find-if (lambda(elem) (string= (get-attribute elem "rel") "http://www.egovpt.org/sdshare/fragmentsfeed")) link-elems))) - (import-snapshots-feed - (get-attribute snapshot-feed-link-elem "href")) - (import-fragments-feed - (get-attribute fragment-feed-link-elem "href")))) + + ;;Process dependencies + (dolist (dependency-elem + (xpath-select-location-path feed-dom + '((*egovpt-ns* "dependency")))) + (let ;;prevent circular dependencies + ((dependent-feed-url + (xpath-fn-string dependency-elem))) + (unless (find dependent-feed-url processed-feed-urls) + (import-tm-feed dependent-feed-url (append processed-feed-urls feed-url))))) + + + (let + ((imported-snapshot-entry + (import-snapshots-feed + (get-attribute snapshot-feed-link-elem "href")))) + (assert imported-snapshot-entry) + (import-fragments-feed + (get-attribute fragment-feed-link-elem "href") + imported-snapshot-entry)))) \ No newline at end of file