;;;; -*- mode: Lisp -*- ;;;; ;;;; $Id: serialization.lisp,v 1.9 2005/01/24 10:04:15 scaekenberghe Exp $ ;;;; ;;;; XML and S-Expression based Serialization for Common Lisp and CLOS ;;;; ;;;; Copyright (C) 2003, 2004 Sven Van Caekenberghe, Beta Nine BVBA. ;;;; ;;;; You are granted the rights to distribute and use this software ;;;; as governed by the terms of the Lisp Lesser General Public License ;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. (in-package :s-serialization) ;;; Public API (defgeneric serializable-slots (object) (:documentation "Return a list of slot names that need serialization")) (defun serialize-xml (object stream &optional (serialization-state (make-serialization-state))) "Write a serialized version of object to stream using XML, optionally reusing a serialization-state" (reset serialization-state) (serialize-xml-internal object stream serialization-state)) (defun serialize-sexp (object stream &optional (serialization-state (make-serialization-state))) "Write a serialized version of object to stream using s-expressions, optionally reusing a serialization-state" (reset serialization-state) (serialize-sexp-internal object stream serialization-state)) (defgeneric serialize-xml-internal (object stream serialization-state) (:documentation "Write a serialized version of object to stream using XML")) (defgeneric serialize-sexp-internal (object stream serialization-state) (:documentation "Write a serialized version of object to stream using s-expressions")) (defun deserialize-xml (stream &optional (serialization-state (make-serialization-state))) "Read and return an XML serialized version of a lisp object from stream, optionally reusing a serialization state" (reset serialization-state) (let ((*deserialized-objects* (get-hashtable serialization-state))) (declare (special *deserialized-objects*)) (car (s-xml:start-parse-xml stream (get-xml-parser-state serialization-state))))) (defun deserialize-sexp (stream &optional (serialization-state (make-serialization-state))) "Read and return an s-expression serialized version of a lisp object from stream, optionally reusing a serialization state" (reset serialization-state) (let ((sexp (read stream nil :eof))) (if (eq sexp :eof) nil (deserialize-sexp-internal sexp (get-hashtable serialization-state))))) (defun make-serialization-state () "Create a reusable serialization state to pass as optional argument to [de]serialize-xml" (make-instance 'serialization-state)) (defgeneric reset-known-slots (serialization-state &optional class) (:documentation "Clear the caching of known slots for class, or for all classes if class is nil")) ;;; Implementation ;; State and Support (defclass serialization-state () ((xml-parser-state :initform nil) (counter :accessor get-counter :initform 0) (hashtable :reader get-hashtable :initform (make-hash-table :test 'eq :size 1024 :rehash-size 2.0)) (known-slots :initform (make-hash-table)))) (defmethod get-xml-parser-state ((serialization-state serialization-state)) (with-slots (xml-parser-state) serialization-state (or xml-parser-state (setf xml-parser-state (make-instance 's-xml:xml-parser-state :new-element-hook #'deserialize-xml-new-element :finish-element-hook #'deserialize-xml-finish-element :text-hook #'deserialize-xml-text))))) (defmethod reset ((serialization-state serialization-state)) (with-slots (hashtable counter) serialization-state (clrhash hashtable) (setf counter 0))) (defmethod reset-known-slots ((serialization-state serialization-state) &optional class) (with-slots (known-slots) serialization-state (if class (remhash (if (symbolp class) class (class-name class)) known-slots) (clrhash known-slots)))) (defmethod known-object-id ((serialization-state serialization-state) object) (gethash object (get-hashtable serialization-state))) (defmethod set-known-object ((serialization-state serialization-state) object) (setf (gethash object (get-hashtable serialization-state)) (incf (get-counter serialization-state)))) ;; when printing symbols we always add the package and treat the symbol as internal ;; so that the serialization is independent of future change in export status ;; we handling symbols in the common-lisp and keyword package more efficiently ;; some hacking to handle unprintable symbols is involved (defconstant +cl-package+ (find-package :cl)) (defconstant +keyword-package+ (find-package :keyword)) (defun print-symbol-xml (symbol stream) (let ((package (symbol-package symbol)) (name (prin1-to-string symbol))) (cond ((eq package +cl-package+) (write-string "CL:" stream)) ((eq package +keyword-package+) (write-char #\: stream)) (t (s-xml:print-string-xml (package-name package) stream) (write-string "::" stream))) (if (char= (char name (1- (length name))) #\|) (s-xml:print-string-xml name stream :start (position #\| name)) (s-xml:print-string-xml name stream :start (1+ (or (position #\: name :from-end t) -1)))))) (defun print-symbol (symbol stream) (let ((package (symbol-package symbol)) (name (prin1-to-string symbol))) (cond ((eq package +cl-package+) (write-string "CL:" stream)) ((eq package +keyword-package+) (write-char #\: stream)) (t (s-xml:print-string-xml (package-name package) stream) (write-string "::" stream))) (if (char= (char name (1- (length name))) #\|) (write-string name stream :start (position #\| name)) (write-string name stream :start (1+ (or (position #\: name :from-end t) -1)))))) (defmethod serializable-slots ((object structure-object)) #+openmcl (let* ((sd (gethash (class-name (class-of object)) ccl::%defstructs%)) (slots (if sd (ccl::sd-slots sd)))) (mapcar #'car (if (symbolp (caar slots)) slots (cdr slots)))) #+cmu (mapcar #'pcl:slot-definition-name (pcl:class-slots (class-of object))) #+sbcl (mapcar #'sb-pcl:slot-definition-name (sb-pcl:class-slots (class-of object))) #+lispworks (structure:structure-class-slot-names (class-of object)) #+allegro (mapcar #'mop:slot-definition-name (mop:class-slots (class-of object))) #+sbcl (mapcar #'sb-mop:slot-definition-name (sb-mop:class-slots (class-of object))) #-(or openmcl cmu lispworks allegro sbcl) (error "not yet implemented")) (defmethod serializable-slots ((object standard-object)) #+openmcl (mapcar #'ccl:slot-definition-name (#-openmcl-native-threads ccl:class-instance-slots #+openmcl-native-threads ccl:class-slots (class-of object))) #+cmu (mapcar #'pcl:slot-definition-name (pcl:class-slots (class-of object))) #+sbcl (mapcar #'sb-pcl:slot-definition-name (sb-pcl:class-slots (class-of object))) #+lispworks (mapcar #'hcl:slot-definition-name (hcl:class-slots (class-of object))) #+allegro (mapcar #'mop:slot-definition-name (mop:class-slots (class-of object))) #+sbcl (mapcar #'sb-mop:slot-definition-name (sb-mop:class-slots (class-of object))) #-(or openmcl cmu lispworks allegro sbcl) (error "not yet implemented")) (defmethod get-serializable-slots ((serialization-state serialization-state) object) (with-slots (known-slots) serialization-state (let* ((class (class-name (class-of object))) (slots (gethash class known-slots))) (when (not slots) (setf slots (serializable-slots object)) (setf (gethash class known-slots) slots)) slots))) ;; Serializers (defmethod serialize-xml-internal ((object integer) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (prin1 object stream) (write-string "" stream)) (defmethod serialize-xml-internal ((object ratio) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (prin1 object stream) (write-string "" stream)) (defmethod serialize-xml-internal ((object float) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (prin1 object stream) (write-string "" stream)) (defmethod serialize-xml-internal ((object complex) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (prin1 object stream) (write-string "" stream)) (defmethod serialize-sexp-internal ((object number) stream serialize-sexp-internal) (declare (ignore serialize-sexp-internal)) (prin1 object stream)) (defmethod serialize-xml-internal ((object null) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream)) (defmethod serialize-xml-internal ((object (eql 't)) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream)) (defmethod serialize-xml-internal ((object string) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (s-xml:print-string-xml object stream) (write-string "" stream)) (defmethod serialize-xml-internal ((object character) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (s-xml:print-string-xml (princ-to-string object) stream) (write-string "" stream)) (defmethod serialize-xml-internal ((object symbol) stream serialization-state) (declare (ignore serialization-state)) (write-string "" stream) (print-symbol-xml object stream) (write-string "" stream)) (defmethod serialize-sexp-internal ((object null) stream serialization-state) (declare (ignore serialization-state)) (write-string "NIL" stream)) (defmethod serialize-sexp-internal ((object (eql 't)) stream serialization-state) (declare (ignore serialization-state)) (write-string "T" stream)) (defmethod serialize-sexp-internal ((object string) stream serialization-state) (declare (ignore serialization-state)) (prin1 object stream)) (defmethod serialize-sexp-internal ((object character) stream serialization-state) (declare (ignore serialization-state)) (prin1 object stream)) (defmethod serialize-sexp-internal ((object symbol) stream serialization-state) (declare (ignore serialization-state)) (print-symbol object stream)) (defun sequence-type-and-length(sequence) (if (listp sequence) (handler-case (let ((length (list-length sequence))) (if length (values :proper-list length) (values :circular-list nil))) (type-error () (values :dotted-list nil))) (values :proper-sequence (length sequence)))) (defmethod serialize-xml-internal ((object sequence) stream serialization-state) (flet ((proper-sequence (length) (let ((id (set-known-object serialization-state object))) (write-string "" stream) (map nil #'(lambda (element) (serialize-xml-internal element stream serialization-state)) object) (write-string "" stream))) (improper-list () (let ((id (set-known-object serialization-state object))) (write-string "" stream) (serialize-xml-internal (car object) stream serialization-state) (write-char #\Space stream) (serialize-xml-internal (cdr object) stream serialization-state) (write-string "" stream)))) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "" stream)) (multiple-value-bind (seq-type length) (sequence-type-and-length object) (ecase seq-type ((:proper-sequence :proper-list) (proper-sequence length)) ((:dotted-list :circular-list) (improper-list)))))))) (defmethod serialize-sexp-internal ((object sequence) stream serialization-state) (flet ((proper-sequence (length) (let ((id (set-known-object serialization-state object))) (write-string "(:SEQUENCE " stream) (prin1 id stream) (write-string " :CLASS " stream) (print-symbol (etypecase object (list 'list) (vector 'vector)) stream) (write-string " :SIZE " stream) (prin1 length stream) (unless (zerop length) (write-string " :ELEMENTS (" stream) (map nil #'(lambda (element) (write-string " " stream) (serialize-sexp-internal element stream serialization-state)) object)) (write-string " ) )" stream))) (improper-list () (let ((id (set-known-object serialization-state object))) (write-string "(:CONS " stream) (prin1 id stream) (write-char #\Space stream) (serialize-sexp-internal (car object) stream serialization-state) (write-char #\Space stream) (serialize-sexp-internal (cdr object) stream serialization-state) (write-string " ) " stream)))) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "(:REF . " stream) (prin1 id stream) (write-string ")" stream)) (multiple-value-bind (seq-type length) (sequence-type-and-length object) (ecase seq-type ((:proper-sequence :proper-list) (proper-sequence length)) ((:dotted-list :circular-list) (improper-list)))))))) (defmethod serialize-xml-internal ((object hash-table) stream serialization-state) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "" stream)) (progn (setf id (set-known-object serialization-state object)) (write-string "" stream) (maphash #'(lambda (key value) (write-string "" stream) (serialize-xml-internal key stream serialization-state) (write-string "" stream) (serialize-xml-internal value stream serialization-state) (princ "" stream)) object) (write-string "" stream))))) (defmethod serialize-sexp-internal ((object hash-table) stream serialization-state) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "(:REF . " stream) (prin1 id stream) (write-string ")" stream)) (let ((count (hash-table-count object))) (setf id (set-known-object serialization-state object)) (write-string "(:HASH-TABLE " stream) (prin1 id stream) (write-string " :TEST " stream) (print-symbol (hash-table-test object) stream) (write-string " :SIZE " stream) (prin1 (hash-table-size object) stream) (write-string " :REHASH-SIZE " stream) (prin1 (hash-table-rehash-size object) stream) (write-string " :REHASH-THRESHOLD " stream) (prin1 (hash-table-rehash-threshold object) stream) (unless (zerop count) (write-string " :ENTRIES (" stream) (maphash #'(lambda (key value) (write-string " (" stream) (serialize-sexp-internal key stream serialization-state) (write-string " . " stream) (serialize-sexp-internal value stream serialization-state) (princ ")" stream)) object)) (write-string " ) )" stream))))) (defmethod serialize-xml-internal ((object structure-object) stream serialization-state) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "" stream)) (progn (setf id (set-known-object serialization-state object)) (write-string "" stream) (mapc #'(lambda (slot) (write-string "" stream) (serialize-xml-internal (slot-value object slot) stream serialization-state) (write-string "" stream)) (get-serializable-slots serialization-state object)) (write-string "" stream))))) (defmethod serialize-sexp-internal ((object structure-object) stream serialization-state) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "(:REF . " stream) (prin1 id stream) (write-string ")" stream)) (let ((serializable-slots (get-serializable-slots serialization-state object))) (setf id (set-known-object serialization-state object)) (write-string "(:STRUCT " stream) (prin1 id stream) (write-string " :CLASS " stream) (print-symbol (class-name (class-of object)) stream) (when serializable-slots (write-string " :SLOTS (" stream) (mapc #'(lambda (slot) (write-string " (" stream) (print-symbol slot stream) (write-string " . " stream) (serialize-sexp-internal (slot-value object slot) stream serialization-state) (write-string ")" stream)) serializable-slots)) (write-string " ) )" stream))))) (defmethod serialize-xml-internal ((object standard-object) stream serialization-state) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "" stream)) (progn (setf id (set-known-object serialization-state object)) (write-string "" stream) (loop :for slot :in (get-serializable-slots serialization-state object) :do (when (slot-boundp object slot) (write-string "" stream) (serialize-xml-internal (slot-value object slot) stream serialization-state) (write-string "" stream))) (write-string "" stream))))) (defmethod serialize-sexp-internal ((object standard-object) stream serialization-state) (let ((id (known-object-id serialization-state object))) (if id (progn (write-string "(:REF . " stream) (prin1 id stream) (write-string ")" stream)) (let ((serializable-slots (get-serializable-slots serialization-state object))) (setf id (set-known-object serialization-state object)) (write-string "(:OBJECT " stream) (prin1 id stream) (write-string " :CLASS " stream) (print-symbol (class-name (class-of object)) stream) (when serializable-slots (princ " :SLOTS (" stream) (loop :for slot :in serializable-slots :do (when (slot-boundp object slot) (write-string " (" stream) (print-symbol slot stream) (write-string " . " stream) (serialize-sexp-internal (slot-value object slot) stream serialization-state) (write-string ")" stream)))) (write-string " ) )" stream))))) ;;; Deserialize CLOS instances and Lisp primitives from the XML representation (defun get-attribute-value (name attributes) (cdr (assoc name attributes :test #'eq))) (defun deserialize-xml-new-element (name attributes seed) (declare (ignore seed) (special *deserialized-objects*)) (case name (:sequence (let ((id (parse-integer (get-attribute-value :id attributes))) (class (read-from-string (get-attribute-value :class attributes))) (size (parse-integer (get-attribute-value :size attributes)))) (setf (gethash id *deserialized-objects*) (make-sequence class size)))) (:object (let ((id (parse-integer (get-attribute-value :id attributes))) (class (read-from-string (get-attribute-value :class attributes)))) (setf (gethash id *deserialized-objects*) (make-instance class)))) (:cons (setf (gethash (parse-integer (get-attribute-value :id attributes)) *deserialized-objects*) (cons nil nil))) (:struct (let ((id (parse-integer (get-attribute-value :id attributes))) (class (read-from-string (get-attribute-value :class attributes)))) (setf (gethash id *deserialized-objects*) (funcall (intern (concatenate 'string "MAKE-" (symbol-name class)) (symbol-package class)))))) (:hash-table (let ((id (parse-integer (get-attribute-value :id attributes))) (test (read-from-string (get-attribute-value :test attributes))) (size (parse-integer (get-attribute-value :size attributes)))) (setf (gethash id *deserialized-objects*) (make-hash-table :test test :size size))))) '()) (defun deserialize-xml-finish-element (name attributes parent-seed seed) (declare (special *deserialized-objects*)) (cons (case name (:int (parse-integer seed)) ((:float :ratio :complex :symbol) (read-from-string seed)) (:null nil) (:true t) (:string (or seed "")) (:character (char seed 0)) (:key (car seed)) (:value (car seed)) (:entry (nreverse seed)) (:slot (let ((name (read-from-string (get-attribute-value :name attributes)))) (cons name (car seed)))) (:sequence (let* ((id (parse-integer (get-attribute-value :id attributes))) (sequence (gethash id *deserialized-objects*))) (map-into sequence #'identity (nreverse seed)))) (:cons (let* ((id (parse-integer (get-attribute-value :id attributes))) (cons-pair (gethash id *deserialized-objects*))) (rplaca cons-pair (second seed)) (rplacd cons-pair (first seed)))) (:object (let* ((id (parse-integer (get-attribute-value :id attributes))) (object (gethash id *deserialized-objects*))) (dolist (pair seed object) (when (slot-exists-p object (car pair)) (setf (slot-value object (car pair)) (cdr pair)))))) (:struct (let* ((id (parse-integer (get-attribute-value :id attributes))) (object (gethash id *deserialized-objects*))) (dolist (pair seed object) (when (slot-exists-p object (car pair)) (setf (slot-value object (car pair)) (cdr pair)))))) (:hash-table (let* ((id (parse-integer (get-attribute-value :id attributes))) (hash-table (gethash id *deserialized-objects*))) (dolist (pair seed hash-table) (setf (gethash (car pair) hash-table) (cadr pair))))) (:ref (let ((id (parse-integer (get-attribute-value :id attributes)))) (gethash id *deserialized-objects*)))) parent-seed)) (defun deserialize-xml-text (string seed) (declare (ignore seed)) string) (defun deserialize-sexp-internal (sexp deserialized-objects) (if (atom sexp) sexp (ecase (first sexp) (:sequence (destructuring-bind (id &key class size elements) (rest sexp) (let ((sequence (make-sequence class size))) (setf (gethash id deserialized-objects) sequence) (map-into sequence #'(lambda (x) (deserialize-sexp-internal x deserialized-objects)) elements)))) (:hash-table (destructuring-bind (id &key test size rehash-size rehash-threshold entries) (rest sexp) (let ((hash-table (make-hash-table :size size :test test :rehash-size rehash-size :rehash-threshold rehash-threshold))) (setf (gethash id deserialized-objects) hash-table) (dolist (entry entries) (setf (gethash (deserialize-sexp-internal (first entry) deserialized-objects) hash-table) (deserialize-sexp-internal (rest entry) deserialized-objects))) hash-table))) (:object (destructuring-bind (id &key class slots) (rest sexp) (let ((object (make-instance class))) (setf (gethash id deserialized-objects) object) (dolist (slot slots) (when (slot-exists-p object (first slot)) (setf (slot-value object (first slot)) (deserialize-sexp-internal (rest slot) deserialized-objects)))) object))) (:struct (destructuring-bind (id &key class slots) (rest sexp) (let ((object (funcall (intern (concatenate 'string "MAKE-" (symbol-name class)) (symbol-package class))))) (setf (gethash id deserialized-objects) object) (dolist (slot slots) (when (slot-exists-p object (first slot)) (setf (slot-value object (first slot)) (deserialize-sexp-internal (rest slot) deserialized-objects)))) object))) (:cons (destructuring-bind (id cons-car cons-cdr) (rest sexp) (let ((conspair (cons nil nil))) (setf (gethash id deserialized-objects) conspair) (rplaca conspair (deserialize-sexp-internal cons-car deserialized-objects)) (rplacd conspair (deserialize-sexp-internal cons-cdr deserialized-objects))))) (:ref (gethash (rest sexp) deserialized-objects))))) ;;;; eof