April 2007 - elephant-cvs@common-lisp.net

[elephant-cvs] CVS elephant/src/elephant
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv23936/src/elephant Modified Files: classindex.lisp controller.lisp Log Message: Fixed test bug; cleaned up get-instances-by-xxx fns to use new map operators --- /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/04/28 02:31:15 1.40 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/04/28 03:07:38 1.41 @@ -389,7 +389,7 @@ ;; USER MAPPING API ;; ====================== -(defun map-class (fn class) +(defun map-class (fn class &key collect) "Perform a map operation over all instances of class. Takes a function of one argument, a class instance" (let* ((class (if (symbolp class) @@ -400,9 +400,9 @@ (declare (ignore k)) (funcall fn v))) (declare (dynamic-extent map-fn)) - (map-btree #'map-fn class-idx)))) + (map-btree #'map-fn class-idx :collect collect)))) -(defun map-inverted-index (fn class index &rest args &key start end value from-end) +(defun map-inverted-index (fn class index &rest args &key start end (value nil value-p) from-end collect) "map-inverted-index maps a function of two variables, taking key and instance, over a subset of class instances in the order defined by the index. Specify the class and index by quoted @@ -431,7 +431,10 @@ (declare (ignore pkey)) (funcall fn key value))) (declare (dynamic-extent wrapper)) - (map-index #'wrapper index :start start :end end :value value :from-end from-end)))) + (if value-p + (map-index #'wrapper index :value value :collect collect) + (map-index #'wrapper index :start start :end end :from-end from-end :collect collect))))) + ;; ================= @@ -454,30 +457,27 @@ nil to start or end indicates, respectively, the lowest or highest value in the index")) + +(defun identity2 (k v) + (declare (ignore k)) + v) + +(defun identity3 (k v pk) + (declare (ignore k pk)) + v) + (defmethod get-instances-by-class ((class symbol)) (get-instances-by-class (find-class class))) (defmethod get-instances-by-class ((class persistent-metaclass)) - (let ((instances nil)) - (flet ((accum (c) - (declare (dynamic-extent c)) - (push c instances))) - (map-class #'accum class) - (nreverse instances)))) + (map-class #'identity class :collect t)) (defmethod get-instances-by-value ((class symbol) slot-name value) (get-instances-by-value (find-class class) slot-name value)) (defmethod get-instances-by-value ((class persistent-metaclass) slot-name value) (declare (type (or string symbol) slot-name)) - (let ((instances nil)) - (declare (type list instances)) - (flet ((collector (k v pk) - (declare (ignore k pk)) - (push v instances))) - (declare (dynamic-extent collector)) - (map-index #'collector (find-inverted-index class slot-name) :value value)) - (nreverse instances))) + (map-inverted-index #'identity2 class slot-name :value value :collect t)) (defmethod get-instance-by-value ((class symbol) slot-name value) (let ((list (get-instances-by-value (find-class class) slot-name value))) @@ -495,15 +495,7 @@ (defmethod get-instances-by-range ((class persistent-metaclass) idx-name start end) (declare (type (or fixnum null) start end) (type symbol idx-name)) - (let ((instances nil)) - (declare (type list instances)) - (flet ((collector (k v pk) - (declare (ignore k pk)) - (push v instances))) - (declare (dynamic-extent collector)) - (map-index #'collector (find-inverted-index class idx-name) - :start start :end end)) - (nreverse instances))) + (map-inverted-index #'identity2 class idx-name :start start :end end :collect t)) (defun drop-instances (instances &key (sc *store-controller*)) "Removes a list of persistent objects from all class indices --- /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/04/28 02:31:15 1.51 +++ /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/04/28 03:07:38 1.52 @@ -220,7 +220,7 @@ "Reset the instance cache (flush object lookups). Useful for testing. Does not reclaim existing objects so there will be duplicate instances with identical functionality" - (ele-with-lock ((instance-cache-lock sc)) + (ele-with-fast-lock ((instance-cache-lock sc)) (setf (instance-cache sc) (make-cache-table :test 'eql))))

1 0

[elephant-cvs] CVS elephant/tests
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/tests In directory clnet:/tmp/cvs-serv16753/tests Modified Files: elephant-tests.lisp testconditions.lisp Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/tests/elephant-tests.lisp 2007/03/30 14:34:35 1.31 +++ /project/elephant/cvsroot/elephant/tests/elephant-tests.lisp 2007/04/28 02:31:31 1.32 @@ -300,9 +300,15 @@ (defmacro signals-condition (&body body) `(handler-case - (progn ,@body) - (condition () t) - (:no-error (&rest rest) (declare (ignore rest)) nil))) + (progn ,@body) + (condition () t) + (:no-error (&rest rest) (declare (ignore rest)) nil))) + +(defmacro signals-specific-condition ((condition) &body body) + `(handler-case + (progn ,@body) + (,condition () t) + (:no-error (&rest rest) (declare (ignore rest)) nil))) (defmacro signals-error (&body body) `(handler-case --- /project/elephant/cvsroot/elephant/tests/testconditions.lisp 2007/04/22 03:02:35 1.2 +++ /project/elephant/cvsroot/elephant/tests/testconditions.lisp 2007/04/28 02:31:31 1.3 @@ -28,8 +28,17 @@ (values (is-not-null (add-to-root 'inst1 inst1 :sc sc1)) (is-not-null (add-to-root 'inst2 inst2 :sc sc2)) - (signals-condition (add-to-root 'inst1 inst1 :sc2)) - (signals-condition (add-to-root 'inst2 inst2 :sc1)))) + (signals-specific-condition (cross-reference-error) + (add-to-root 'inst1 inst1 :sc sc2)) + (signals-specific-condition (cross-reference-error) + (add-to-root 'inst2 inst2 :sc sc1)))) (close-store sc1) (close-store sc2)))) - t t t t) \ No newline at end of file + t t t t) + +(deftest unindexed-class-condition + (let* ((sc (open-store *test-spec-primary* :recover t :deadlock-detect nil)) + (inst (make-instance 'pfoo :slot1 1 :sc sc))) + (signals-specific-condition (persistent-class-not-indexed) + (find-class-index 'pfoo :sc sc))) + t) \ No newline at end of file

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv16753/src/elephant Modified Files: classindex.lisp collections.lisp controller.lisp package.lisp transactions.lisp Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/04/24 12:58:10 1.39 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/04/28 02:31:15 1.40 @@ -84,11 +84,23 @@ (let ((class (find-class class-name nil))) (when class (indexed class)))) +(define-condition persistent-class-not-indexed (error) + ((class-obj :initarg :class :initarg nil :reader unindexed-class-obj))) + +(defun signal-class-not-indexed (class) + (cerror "Ignore and continue?" + 'persistent-class-not-indexed + :format-control "Class ~A is not enabled for indexing" + :format-arguments (list (class-name class)) + :class class)) + +;; (define-condition + (defmethod find-class-index ((class persistent-metaclass) &key (sc *store-controller*) (errorp t)) (ensure-finalized class) (if (not (indexed class)) (when errorp - (error "Class ~A is not an indexed class" class)) + (signal-class-not-indexed class)) (if (class-index-cached? class) (%index-cache class) ;; we've got a cached reference, just return it (multiple-value-bind (btree found) @@ -110,31 +122,26 @@ (synchronize-class-to-store class :sc sc :method method) btree)) -(define-condition persistent-class-not-indexed (error) - ((class-obj :initarg :class :initarg nil :reader unindexed-class-obj))) - (defun cache-new-class-index (class sc) "If not cached or persistent then this is a new class, make the new index" (if (indexed class) (enable-class-indexing class (indexing-record-slots (indexed-record class)) :sc sc) - (signal 'persistent-class-not-indexed - :class class - :format-control "Class ~A is not enabled for indexing" - :format-arguments (list (class-name class))))) + (signal-class-not-indexed class))) (defmethod find-inverted-index ((class symbol) slot &key (null-on-fail nil)) (find-inverted-index (find-class class) slot :null-on-fail null-on-fail)) (defmethod find-inverted-index ((class persistent-metaclass) slot &key (null-on-fail nil)) - (let* ((cidx (find-class-index class)) + (let* ((cidx (find-class-index class :errorp (not null-on-fail))) (idx (or (get-index cidx slot) (get-index cidx (make-derived-name slot))))) (if idx idx (if null-on-fail nil - (error "Inverted index ~A not found for class ~A with - persistent slots: ~A" slot (class-name class) (car (%persistent-slots class))))))) + (cerror "Ignore and continue?" + "Inverted index ~A not found for class ~A with persistent slots: ~A" + slot (class-name class) (car (%persistent-slots class))))))) (defmethod find-inverted-index-names ((class persistent-metaclass)) (let ((names nil)) --- /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/27 13:32:17 1.27 +++ /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/28 02:31:15 1.28 @@ -312,6 +312,7 @@ primary key greater or equal to the pkey argument. Returns has-tuple / secondary key / value / primary key.")) + (defgeneric cursor-next-dup (cursor) (:documentation "Move to the next duplicate element (with the same key.) @@ -322,11 +323,6 @@ "Move to the next non-duplicate element (with different key.) Returns has-pair key value.")) -(defgeneric cursor-prev-nodup (cursor) - (:documentation - "Move to the previous non-duplicate element (with -different key.) Returns has-pair key value.")) - (defgeneric cursor-pnext-dup (cursor) (:documentation "Move to the next duplicate element (with the same key.) @@ -338,12 +334,53 @@ key.) Returns has-tuple / secondary key / value / primary key.")) + +(defgeneric cursor-prev-dup (cursor) + (:documentation + "Move to the previous duplicate element (with the same key.) +Returns has-pair key value.")) + +(defmethod cursor-prev-dup ((cur cursor)) + "Default implementation. Plan is to update both backends when BDB 4.6 comes out" + (when (cursor-initialized-p cur) + (multiple-value-bind (exists? skey-cur) + (cursor-current cur) + (declare (ignore exists?)) + (multiple-value-bind (exists? skey value) + (cursor-prev cur) + (if (lisp-compare-equal skey-cur skey) + (values exists? skey value) + (setf (cursor-initialized-p cur) nil)))))) + +(defgeneric cursor-prev-nodup (cursor) + (:documentation + "Move to the previous non-duplicate element (with +different key.) Returns has-pair key value.")) + +(defgeneric cursor-pprev-dup (cursor) + (:documentation + "Move to the previous duplicate element (with the same key.) +Returns has-tuple / secondary key / value / primary key.")) + +(defmethod cursor-pprev-dup ((cur cursor)) + "Default implementation. Plan is to update both backends when BDB 4.6 comes out" + (when (cursor-initialized-p cur) + (multiple-value-bind (exists? skey-cur) + (cursor-current cur) + (declare (ignore exists?)) + (multiple-value-bind (exists? skey value pkey) + (cursor-pprev cur) + (if (lisp-compare-equal skey-cur skey) + (values exists? skey value pkey) + (setf (cursor-initialized-p cur) nil)))))) + (defgeneric cursor-pprev-nodup (cursor) (:documentation "Move to the previous non-duplicate element (with different key.) Returns has-tuple / secondary key / value / primary key.")) + (defmacro with-btree-cursor ((var bt) &body body) "Macro which opens a named cursor on a BTree (primary or not), evaluates the forms, then closes the cursor." @@ -439,6 +476,7 @@ start end)) (let ((sc (get-con index)) (end (or value end)) + (from-end (and from-end (not value-set-p))) (results nil)) (flet ((collector (k v pk) (push (funcall fn k v pk) results))) @@ -454,11 +492,12 @@ (lisp-compare-equal key start)) (lisp-compare<= key end)))) (value-increment () - ;; Step to the next key value + ;; Step to the next key value; + ;; from-end duplicate cursor is already there (if from-end - (pprev-hack cur) + (cursor-current cur) (cursor-pnext-nodup cur))) - (next-value () + (map-values () ;; Handle the next key value (multiple-value-bind (exists? skey val pkey) (value-increment) @@ -468,18 +507,23 @@ (map-duplicates skey)) (return-from map-index (nreverse results))))) + (next-duplicate (key) + (if from-end + (pprev-dup-hack cur key) + (cursor-pnext-dup cur))) (map-duplicates (key) ;; Map all duplicates for key value (multiple-value-bind (exists? skey val pkey) - (cursor-pnext-dup cur) + (next-duplicate key) (if exists? (progn (funcall fn skey val pkey) (map-duplicates key)) (progn - (cursor-pset-range cur key) - (next-value)))))) - (declare (dynamic-extent (function next-value) (function next-value-increment) + (unless from-end + (cursor-pset cur key)) + (map-values)))))) + (declare (dynamic-extent (function map-values) (function next-duplicate) (function continue-p) (function map-duplicates))) (multiple-value-bind (exists? skey val pkey) (cond (value-set-p @@ -487,9 +531,9 @@ ((and (not from-end) (null start)) (cursor-pfirst cur)) ((and from-end (null end)) - (cursor-last-range-hack cur)) + (cursor-plast cur)) (t (if from-end - (cursor-pset-range cur end) + (pset-range-for-descending cur end) (cursor-pset-range cur start)))) (if (and exists? (continue-p skey)) (progn @@ -497,23 +541,24 @@ (map-duplicates skey)) nil))))))))) -(defun pprev-hack (cur) - "Get the first duplicate instance of the prior value off the current cursor" - (let ((e? (cursor-pprev-nodup cur))) - (when e? - (let ((e? (cursor-pprev-nodup cur))) - (if e? - (cursor-pnext cur) - (cursor-pfirst cur)))))) - -(defun cursor-last-range-hack (cur) - "Get the first duplicate instance of the last value of the cursor's index" - (let ((e? (cursor-plast cur))) - (when e? - (let ((e? (cursor-pprev-nodup cur))) - (if e? - (cursor-pnext cur) - (cursor-pfirst cur)))))) +(defun pset-range-for-descending (cur end) + (if (cursor-pset cur end) + (progn + (cursor-next-nodup cur) + (cursor-pprev cur)) + (progn + (cursor-pset-range cur end) + (cursor-pprev cur)))) + +(defun pprev-dup-hack (cur key) + "Go back one step in a duplicate set, returns nil + if previous element is a different key. More efficient than + the current default implementation of cursor-pprev-dup" + (multiple-value-bind (exists? skey value pkey) + (cursor-pprev cur) + (when (lisp-compare-equal key skey) + (values exists? key value pkey)))) + ;; =============================== ;; Some generic utility functions --- /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/04/25 02:28:01 1.50 +++ /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/04/28 02:31:15 1.51 @@ -136,7 +136,7 @@ (defun initialize-user-parameters () (loop for (keyword variable) in *user-configurable-parameters* do (awhen (get-user-configuration-parameter keyword) - (setf variable it)))) + (setq variable it)))) ;; ;; COMMON STORE CONTROLLER FUNCTIONALITY @@ -165,7 +165,7 @@ "This is an instance cache and part of the metaclass protocol. Data stores should not override the default behavior.") - (instance-cache-lock :accessor instance-cache-lock :initform (ele-make-lock) + (instance-cache-lock :accessor instance-cache-lock :initform (ele-make-fast-lock) :documentation "Protection for updates to the cache from multiple threads. Do not override.") @@ -202,14 +202,16 @@ (defun cache-instance (sc obj) "Cache a persistent object with the controller." (declare (type store-controller sc)) - (ele-with-lock ((instance-cache-lock sc)) + (ele-with-fast-lock ((instance-cache-lock sc)) (setf (get-cache (oid obj) (instance-cache sc)) obj))) (defun get-cached-instance (sc oid class-name) "Get a cached instance, or instantiate!" (declare (type store-controller sc) (type fixnum oid)) - (let ((obj (get-cache oid (instance-cache sc)))) + (let ((obj + (ele-with-fast-lock ((instance-cache-lock sc)) + (get-cache oid (instance-cache sc))))) (if obj obj ;; Should get cached since make-instance calls cache-instance (make-instance class-name :from-oid oid :sc sc)))) --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/27 13:32:17 1.34 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/28 02:31:16 1.35 @@ -222,12 +222,12 @@ #:cursor #:secondary-cursor #:make-cursor #:make-simple-cursor #:cursor-close #:cursor-duplicate #:cursor-current #:cursor-first #:cursor-last #:cursor-next #:cursor-next-dup - #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup + #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup #:cursor-prev-dup #:cursor-set #:cursor-set-range #:cursor-get-both #:cursor-get-both-range #:cursor-delete #:cursor-put #:cursor-pcurrent #:cursor-pfirst #:cursor-plast #:cursor-pnext #:cursor-pnext-dup #:cursor-pnext-nodup - #:cursor-pprev #:cursor-pprev-nodup #:cursor-pset + #:cursor-pprev #:cursor-pprev-dup #:cursor-pprev-nodup #:cursor-pset #:cursor-pset-range #:cursor-pget-both #:cursor-pget-both-range #:cursor-initialized-p @@ -267,6 +267,7 @@ ;; Various error conditions #:cross-reference-error #:controller-lost-error + #:persistent-class-not-indexed #:map-class-query #:get-query-instances --- /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/27 13:32:17 1.12 +++ /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/28 02:31:16 1.13 @@ -120,7 +120,7 @@ If nested, the backend must support nested transactions." (let ((sc (gensym))) `(let ((,sc ,store-controller)) - (funcall #'execute-transaction ,store-controller + (funcall #'execute-transaction ,sc (lambda () ,@body) :parent (if (owned-txn-p ,sc ,parent) (transaction-object ,parent) @@ -146,10 +146,9 @@ (,sc ,store-controller)) (if (owned-txn-p ,sc ,parent) (funcall ,txn-fn) - (funcall #'execute-transaction ,store-controller + (funcall #'execute-transaction ,sc ,txn-fn :parent nil - :transaction nil :retries ,retries ,@(remove-keywords '(:store-controller :parent :transaction :retries) keyargs))))))

1 0

[elephant-cvs] CVS elephant/doc/oldfiles
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/doc/oldfiles In directory clnet:/tmp/cvs-serv16753/doc/oldfiles Added Files: INSTALL NEWS NOTES TODO TUTORIAL Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/doc/oldfiles/INSTALL 2007/04/28 02:31:14 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/INSTALL 2007/04/28 02:31:14 1.1 ------------ Requirements ------------ Supported Lisps: CMUCL 19a Linux SBCL 0.9.17/1.0+ Linux / Mac OSX Allegro CL 7.0/8.0 Linux / Mac OSX OpenMCL 0.14.2 LispWorks (port in-progress) Lisp libraries: ASDF - http://www.cliki.net/asdf UFFI 1.5.4+ - http://uffi.b9.com/ A Backend Database: 1) Oracle Berkeley DB 4.5 - http://www.oracle.com/database/berkeley-db.html 2) CLSQL - http://clsql.b9.com/ with an appropriate SQL installation. Tested with SQlite3 and Postgresql so far A C compiler, probably gcc or Visual Studio. Presumably you have this if you installed ------------------ Short Instructions ------------------ The new build system should work out of the box on most Un*x platforms that have asdf, uffi and either clsql or Berkeley DB installed in the usual places. Try: (asdf:operate 'asdf:load-op :elephant) Then: (open-store '(<backend> <spec>)) Where <backend> = { :bdb | :clsql } <spec> = { "fresh directory for BDB files" | '(:sqlite3 "db path") | '(:postgresql "db path") This should load all files, including compiling libraries, on most systems. For Win32, see the instructions below. (We'll improve the build process for Win32 if there is demand) ----------------- Long Instructions ----------------- For SBCL, CMUCL, Allegro 8.0+, MCL and CLISP: 0) Unpack Elephant. I put mine in the directory /usr/local/share/common-lisp/elephant-0.6.x/ 1) Install ASDF. Ensure that you have a recent version of ASDF installed as the load process now depends upon it. 2) Install UFFI 3) Install a backend: Either Berkeley DB 4.5, PostGresql, or SQLite 3. ------- SQL ------- For relational database systems, refering the formal documentation other the heading "SQL-BACK-END". ------------- Berkeley 4.5: ------------- (Note: 0.6.0 required BDB 4.3; to upgrade 0.6.0 to 0.6.1, upgrade BDB to 4.5, modify my-config.sexp appropriately then run 0.6.1+; your underlying Berekely DB files will automatically upgrade when the DB is opened. To use 0.6.1, you will have to manually migrate your 0.6.0 database to a fresh database created in 0.6.1) Under Un*x, you may actually already have this installed, though it may be compiled with funny options, so if things don't work you may want to try to start from scratch. FreeBSD has a port for this, as I'm sure do other BSDs (including DarwinPorts/Fink.) Take note of where libdb.so and db.h are installed, usually: /usr/local/BerkeleyDB.4.5/lib/libdb.so and /usr/local/BerkeleyDB.4.5/include/db.h, or /usr/local/lib/db45/libdb.so and /usr/local/include/db45/db.h.) a) Site specific configuration config.sexp Which contains an alist providing string paths pointing to the root of the Berkeley DB distribution :berkeley-db-root, the library to load :berkeley-db-lib and the pthreads library if you're running linux :pthread-lib. For Win32 (directions courtesy of Bill Clementson): --------------------------------------------------- Create an MSVC dll project and add src/db-bdb/libberkeley-db.c, src/db-bdb/libberkeley-db.def and the Berkeley DB libdb43.lib files to the project (should be in the build_win32/release folder) Add the Berkeley DB dbinc include files directory and the build_win32/release directory (where the Berkeley DB install instructions builds the Berkeley DB objects by default) to the build directories for the project Build the Elephant DLL file Since you've statically included libdb43.lib inside libberkeley-db.c, it may or may not be necessary to load libdb43.dll into Lisp (see below.) 4) Compile and load Elephant: The new backend load process should work automatically on Un*x systems but if there are probolems with loading foreign libraries, then you can test your C tools setup with 'make' in the elephant root directory. This will build the common memutils library in src/memutil/libmemutil.so/dylib that all backends require. There is a new two-phase load process. The first requires that you use asdf to load the main elephant front-end: (asdf:operate 'asdf:load-op :elephant) This will load and compile Elephant. This will also automatically load UFFI. When you call (open-store <spec>) inside lisp it will automatically load the remaining dependencies for the specified backend via ASDF. To test the load process explicitly the following asdf files are provided: if you are using Berkeley DB, type: (asdf:operate 'asdf:load-op :ele-bdb) if you are using CL-SQL, type: (asdf:operate 'asdf:load-op :ele-clsql) if you are using SQLite3, type: (asdf:operate 'asdf:load-op :ele-sqlite3) 5) Make the documentation: Execute: make In the doc directory should be build the HTML version of the texinfo files. ------- Testing ------- Elephant uses RT for regression testing, available at: http://www.cliki.net/RT Once RT is installed (asdf:operate 'asdf:load-op :elephant-tests) (in-package :ele-tests) (setf *default-spec* <backend>) Where <backend> = { *testsqlite3-spec* | *testpg-spec* | *testbdb-spec* } (do-backend-tests) This will test the standalone API for your backend. Currently all tests are passing on 0.6.0. There will be a set of migration tests that will be 'ignored' but the final message should indicate no failing tests. This should take less than 5 minutes on decent hardware. The tests are not idempotent, so if you run the tests a second time, they are likely to fail. To avoid this, for example if you are debugging tests, just run the script delscript.sh (or do the equivalent on Win32) in the elephant/tests directory. Elephant allows migration between repositories. To test this: (do-migration-tests *default-spec* <backend>) where <backend> is a different *testXXXXX-spec* variable to test migration to that backend. This should take less than 2 minutes on decent hardware. A backend is considered "green" if it can pass both the backend tests and the migration tests. --- /project/elephant/cvsroot/elephant/doc/oldfiles/NEWS 2007/04/28 02:31:14 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/NEWS 2007/04/28 02:31:14 1.1 April, 2006 - Elephant 0.6.0 released by Robert Read and Ian Eslick. Supports class slot indexing and benefits from a clean refactoring of backends and a host of other small changes. This is a solid BETA release. November 30, 2005 - Elephant 0.3.0 released by the new maintainer, Robert L. Read, providing support for relational database backends, repository migration, and multi-repository operation. As of this release, the documentation provides a lot of information about installation and getting things working; I wouldn't at all claim that it is complete, smooth, or well organized. The more notes I get about the use of Elephant, the more inclined I will be to invest time in improving the documentation. October 7, 2004 - Elephant 0.2.1 released. Thanks to Bill Clementson, Elephant should compile on Win32 now. Also, a few minor fixups. September 19, 2004 - Elephant 0.2 released. This is an BETA release. New features: - Secondary indices and cursors - PPC Darwin OpenMCL / SBCL - Doc strings and improved documentation - An RT-based test suite - many bugfixes This release has been tested on CMUCL 19a, SBCL 0.8.14 and Allegro 6.2 on x86 Linux and FreeBSD, and OpenMCL 0.14.2-p1 and SBCL 0.8.14 on PPC Darwin. September 2, 2004 - The bad news: there was a bug in 0.1 which made OID generation inside of manual transactions deadlock. The good news: this is fixed, and I've added OpenMCL support. So I'm releasing 0.1-p1. August 30, 2004 - Elephant 0.1 was released August 30th, 2004. This is an ALPHA quality release, so claims about correctness, performance and safety should be taken with a grain of salt. This release has been tested on CMUCL 19a, SBCL 0.8.13 and Allegro 6.2 on x86 Linux and FreeBSD. OpenMCL and Lispworks versions will come soon. As a proof of concept I've compiled and run CL-IRC http:://www.common-lisp.net/project/cl-irc making all objects and slots persistent, except for the socket-streams. It runs, and saves everything except for the socket-streams. --- /project/elephant/cvsroot/elephant/doc/oldfiles/NOTES 2007/04/28 02:31:15 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/NOTES 2007/04/28 02:31:15 1.1 ------- GENERAL ------- this has been optimized for use with CMUCL / SBCL / Allegro. OpenMCL has been minimally supported. Lispworks is a target as well but less so as the developers don't have access to it. Theoretically one can port this to any lisp with a decent FFI and MOP. However since those are two of the less standardized bits of Lisp, in practice this might be difficult. >From top to bottom, here are the implementation layers: ELEPHANT package persistent meta-object, persistent collections controller serializer memutils package UFFI / implementation specific stuff libsleepycat.so Sleepycat 4.2/3 While I loath specials, since you can't change the signature of slot accessors, in order to pass parameters to the database / serializer, specials are needed. Also specials will probably play nice with threaded lisps. ----------------------- CLASSES AND METACLASSES ----------------------- Persistent classes which the user defines are declared and instrumented by using the persistent-metaclass. Ideally creating persistent versions of class, slot-defintion, et al would be enough, but in reality various implementations do things in different ways. CMUCL / SBCL: their's a bit of work to make class slot allocation and reader / writer / slot-boundp work right. Allegro: is using slot-boundp instead of slot-boundp-using-class inside of shared-initialize, which necessitates some work. CMUCL doesn't do non-standard allocation types correctly, so we've created our own slot definition keyword :transient. In the future this will change. Andrew will add some notes here in the future. ----------- COLLECTIONS ----------- While we support serializing and persisting a wide class of Lisp data types, there are problems with persisting aggregate types (conses, lists, arrays, objects, hash-tables...) 1) not automatic: there's no way for elephant to know when you've changed a value in an aggregate object, so you have to manually restore it back into the slot to get it saved. example 1: you put a cons into the database. you change it's car. this is not saved unless you resave the cons into the database. example 2: slot-1 of obj A (saved in the database) contains a cons. you change the car of the cons. this is not reflected into the database unless you resave A. 2) merge-conflicts: changing one value and saving an aggregate will write out the whole aggregate, possibly blowing away changes other threads have made behind your back. this is not protected by transactions! 3) consing, non-lazy and expensive (de)serialization: you have to serialize/deserialize the entire aggregate every time you save it or restore it. This is pretty fast all things considered, but it's probably better to use persistent collections. 4) you have to store the entire collection in memory, whereas one of the points of the database to store large collections of objects..... For these and other reasons, we provide a hash-table-like interface to Berkeley BTrees. These have many advantages over ordinary hash-tables from the point of view of persistence. There is a separate table for BTrees. This is because we use a hand coded C function for sorting, which understands a little of the serialized data. It can handle numbers (up to 64-bit bignums -- they are approximated by floats) and strings (case-insensitive for 8-bit, code-point-order for 16-bit Unicode.) It should be fast but we don't want a performance penalty on objects. Secondary indices are mostly handled on the lisp side, because of our weird table layout (see below) and to avoid crossing FFI boundaries. Some unscientific microbenchmarks indicated that there was no performance benefit on CMUCL / SBCL, and only minor benefit (asymptotically nil) on OpenMCL. They have a separate table. Actually two handles are opened on this table: one which is plain, and one which is associated to the primary btree table by a no-op indexing function. Since we maintain the secondary keys ourselves, the associated handle is good for gets / cursor traversals. We use the unassociated handle for updates. ---------- CONTROLLER ---------- The controller is accessed through the special *store-controller*. The controller keeps track of 1) the environment handle 2) the DB handle(s) 3) the instance cache 4) the root object The environment handle and DB handle currently aren't really exposed. Eventually they should be, so that tuning flags can be set on them. OIDs are generated by a bit of C code, which isn't great, nor that safe (to get acceptable performance i use DB_TXN_NOSYNC.) Waiting for Sleepycat 4.3. The instance cache is implemented as a values-weak hash-table. This is a hash-table where the values can be collected, and when they are, the entire key-value entry is deleted. There are implementations of this on the various platforms. The instance cache is there to make deserialization of persistant objects faster. Since we [249 lines skipped] --- /project/elephant/cvsroot/elephant/doc/oldfiles/TODO 2007/04/28 02:31:15 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/TODO 2007/04/28 02:31:15 1.1 [498 lines skipped] --- /project/elephant/cvsroot/elephant/doc/oldfiles/TUTORIAL 2007/04/28 02:31:15 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/TUTORIAL 2007/04/28 02:31:15 1.1 [923 lines skipped]

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv16753/src/db-bdb Modified Files: bdb-transactions.lisp Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/27 13:32:14 1.15 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/28 02:31:15 1.16 @@ -31,15 +31,13 @@ for count fixnum from 1 to retries for success of-type boolean = nil do - (let ((txn - (if transaction transaction - (db-transaction-begin env + (let ((txn (db-transaction-begin env :parent (if parent parent +NULL-VOID+) :degree-2 degree-2 :read-uncommitted read-uncommitted :txn-nosync txn-nosync :txn-nowait txn-nowait - :txn-sync txn-sync)))) + :txn-sync txn-sync))) (declare (type pointer-void txn)) (let (result) (let ((*current-transaction* (make-transaction-record sc txn))

1 0

[elephant-cvs] CVS elephant/doc
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/doc In directory clnet:/tmp/cvs-serv16753/doc Modified Files: copying.texinfo data-store-reference.texinfo elephant.texinfo installation.texinfo reference.texinfo scenarios.texinfo tutorial.texinfo user-guide.texinfo Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/doc/copying.texinfo 2007/03/30 14:34:34 1.4 +++ /project/elephant/cvsroot/elephant/doc/copying.texinfo 2007/04/28 02:31:07 1.5 @@ -8,29 +8,30 @@ @section Elephant Licensing -@b{Elephant}: a persistent metaprotocol and object-oriented database -for Common Lisp. +Elephant is a persistent metaprotocol and object-oriented database for +Common Lisp. Detailed information and distributions can be found at +@uref{http://www.common-lisp.net/project/elephant}. -Homepage: @uref{http://www.common-lisp.net/project/elephant} +The program is released under the following license: @quotation -Elephant users are granted the rights to distribute and use this software -as governed by the terms of the Lisp Lesser GNU Public License -@uref{http://opensource.franz.com/preamble.html}, also known as the LLGPL. +Elephant users are granted the rights to distribute and use this +software as governed by the terms of the Lisp Lesser GNU Public +License @uref{http://opensource.franz.com/preamble.html}, also known +as the LLGPL. @end quotation Copyrights include: @quotation -Copyright (c) 2004 by Andrew Blumberg and Ben Lee - -Copyright (c) 2005-2007 by Robert L. Read - -Copyright (c) 2006-2007 by Ian Eslick +Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. @* +Version 0.5, Copyright @copyright{} 2006 Robert L. Read. @* +Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* +Portions copyright respective contributors (see @file{CREDITS}). @end quotation -Portions of this program (namely the C unicode string sorter) are -derived from IBM's @b{ICU}: @uref{http://oss.software.ibm.com/icu/, +Portions of the program (namely the C unicode string sorter) are +derived from IBM's ICU: @uref{http://oss.software.ibm.com/icu/, ICU Website} whose copyright and license follows below. @quotation @@ -75,6 +76,33 @@ are the property of their respective owners. @end quotation +@section Elephant Manual Copyright and Licensing + +@quotation +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License. +@end quotation + +Copyrights include: + +@quotation +Original Version, Copyright @copyright{} 2004 Ben Lee. @* +Versions 0.5-0.6, Copyright @copyright{} 2006 Robert L. Read. @* +Current Version, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* +@end quotation + +@section 3rd Party Libraries + +Elephant depends on 3rd party lisp libraries. See their respective +distributions for detailed copyright and licensing information. The +following is a brief summary. + +@itemize +@item @b{uffi}: By Kevin Rosenberg, no significant restrictions +@item @b{cl-base64}: By Kevin Rosenberg, no significant restrictions +@item @b{rt}: By Richard Waters, MIT License +@end itemize + @section Data Store Licensing Considerations The Berkeley DB data store is based on the Berkeley DB C library, now @@ -100,5 +128,3 @@ @item MySQL: Dual licensing (similar to BDB), see @uref{http://www.mysql.com/company/legal/licensing/, the MySQL license page} @end itemize - - --- /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/04/27 03:14:55 1.7 +++ /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/04/28 02:31:07 1.8 @@ -34,7 +34,7 @@ * Cursors: DSR Cursors. Traversing BTrees. * Transactions: DSR Transactions. Transaction implementation. * Multithreading Considerations: DSR Multithreading Considerations. Multithreading considerations. -* Foreign Libraries: DSR Foreign Libraries. Using UFFI and ASDF to build or link foreign libraries +@c * Foreign Libraries: DSR Foreign Libraries. Using UFFI and ASDF to build or link foreign libraries @end menu @node DSR Registration @@ -147,8 +147,8 @@ implemented using map-btree and map-index. @itemize -@item @ref{Generic-Function elephant:get-value} -@item @ref{Generic-Function (setf elephant:get-value)} +@item @ref{Generic-Function elephant:get-value} (and @code{(setf get-value)}) +@c @item @ref{Generic-Function (setf elephant:get-value)} @item @ref{Generic-Function elephant:existsp} @item @ref{Generic-Function elephant:remove-kv} @item @ref{Generic-Function elephant:get-index} @@ -168,7 +168,7 @@ @cindex Cursors Data stores must subclass these cursor classes and implement all the -methods described in @ref{Cursors} except @ref{Macro +methods described in @ref{DSR Cursors} except @ref{Macro elephant:with-btree-cursor}. @include includes/class-elephant-cursor.texinfo @@ -220,9 +220,23 @@ @section Multithreading Considerations @cindex Multithreading -Generic locking utility functions +This expands slightly on the multithreading discussion in +@ref{Multi-threaded Applications}. -Variable behavior in multithreading situations +Elephant provides a set of generic locking functions in +(a)code{src/utils/locks.lisp} to help protect any shared structures. +There are standard locking functions (@code{ele-with-lock}) and then +a special locking interface called @code{ele-with-fast-lock} which +on some lisps provides a faster locking option than the standard OS +locks of the basic interface. (i.e. under Allegro this uses +@code{without-interrupts} because Allegro still runs in a single +OS process on all platforms, this is not true of SBCL). + +See the sections on Transaction handling, particularly the dynamic +behavior of @code{*current-transaction*}. Also read up on the +store controller section in the User Guide to better understand the +role of @code{*store-controller*}. At this time there are no other +global variables to worry about. @node DSR Handling Serialization @comment node-name, next, previous, up @@ -253,19 +267,19 @@ @include includes/fun-elephant-data-store-serialize-to-base64-string.texinfo @include includes/fun-elephant-data-store-deserialize-from-base64-string.texinfo -@node DSR Memory Utilities -@comment node-name, next, previous, up -@section Memory utilities -@cindex Memory utilities - -Details about memory utilities here. - -@node DSR Foreign Libraries -@comment node-name, next, previous, up -@section Foreign libraries -@cindex Foreign libraries +@c @node DSR Memory Utilities +@c @comment node-name, next, previous, up +@c @section Memory utilities +@c @cindex Memory utilities + +@c Details about memory utilities here. + +@c @node DSR Foreign Libraries +@c @comment node-name, next, previous, up +@c @section Foreign libraries +@c @cindex Foreign libraries -How foreign libraries are built and used via UFFI. What functions are -in the .asd files or main lisp code to build & load libraries? +@c How foreign libraries are built and used via UFFI. What functions are +@c in the .asd files or main lisp code to build & load libraries? --- /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/04/27 03:14:55 1.10 +++ /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/04/28 02:31:07 1.11 @@ -2,19 +2,27 @@ @c %**start of header @setfilename elephant.info @settitle Elephant User Manual +@setchapternewpage odd @c %**end of header - -@copying -Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. -Version 0.5, Copyright @copyright{} 2006 Robert L. Read. -Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read +@copying @quotation +Elephant System @* +Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. @* +Version 0.5, Copyright @copyright{} 2006 Robert L. Read. @* +Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* +Portions copyright respective contributors (see @file{CREDITS}). @* + +Elephant Manual @* +Original Version, Copyright @copyright{} 2004 Ben Lee. @* +Versions 0.5-0.6, Copyright @copyright{} 2006 Robert L. Read. @* +Current Version, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* + Permission is granted to copy, distribute and/or modify this document -under the terms of the GNU Free Documentation License. Portions of -Elephant are derived from ICU, IBM's Unicode library. Details about -copyright, license and warranty information can be found in -@file{LICENSE} or in the ``Copying'' section of this document. +under the terms of the GNU Free Documentation License. See the +Copyright and License chapter for details about copyright, license and +warranty for this manual and the Elephant system. + @end quotation @end copying --- /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/04/27 03:14:55 1.11 +++ /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/04/28 02:31:07 1.12 @@ -9,9 +9,10 @@ * Requirements:: Supported lisps and required libraries. * Configuring Elephant:: Setting up Elephant and the configuration file. * Loading Elephant:: Loading Elephant and the data store loading protocol. -* Berkeley DB:: Installing support for the Berkeley DB data store +* Berkeley DB:: Installing support for the Berkeley DB data store. * Berkeley DB Example:: An example of installing and running the Berkeley DB data store. -* CL-SQL:: Install and connecting to the CL-SQL data store +* Upgrading Berkeley DB Database:: How to upgrade to a new version of Berkeley DB. +* CL-SQL:: Install and connecting to the CL-SQL data store. * CL-SQL Example:: An example of using the CL-SQL data store. * Elephant on Windows:: More details about running Elephant on Windows * Test Suites:: How to run and interpret the output of the regression test suite @@ -275,6 +276,57 @@ @end lisp in your application. +@node Upgrading Berkeley DB Databases +@comment node-name, next, previous, up +@section Upgrading Berkeley DB Databases + +At regular intervals, Elephant will require a specific version of Berkeley DB +because it does not parse the header files which tend to change with each release. +The patches are usually minor, but sometimes Elephant also depends on new +features of Berkeley DB. In this case, you have to update both the Berkeley DB +database files as well as any data Elephant has built within it. That creates +some special constraints for upgrading databases. + +@subsection Upgrading to 0.9 + +This section outlines how to upgrade from Elephant code base version 0.6.0 +using Berkeley DB 4.3. + +@enumerate +@item Install BDB 4.5 (keep 4.3 around for now) +@item Update my-config.sexp to point to the appropriate BDB 4.5 directories +@item Upgrade your database directory to 4.5 + @itemize + @item Run db43_recover in your 0.6 database + @item Optional: run db43_archive -d to remove all logs not part of a checkpoint + This will make catastrophic recovery impossible, but reduces the amount of data you + have to backup. + @item Backup your db files and remaining logs + @item Run db45_checkpoint -1 in your 0.6 database directory + @end itemize +@item Migrate 0.6 data to a new 0.6.1 database + @itemize + @item Open your old database: @code{(setf sc (open-store '(:BDB "/Users/me/db/ele060/")))} + @item Run upgrade: @code{(upgrade sc '(:BDB "/Users/me/db/ele061/"))} + @end itemize +@item Test your new application and report any bugs that arise to elephant-devel(a)common-lisp.net +@end itemize + +@emph{(NOTE: close-store may fail when closing the old 0.6 database, this is OK.)} + +@emph{(NOTE: 64-bit lisps will not successfully upgrade 32-bit 0.6 databases. Use a 32-bit + version of your lisp to update to 0.6.1 and then open that database in your 64-bit + lisp. There should be no compatibility problems. Best to test your application on + a 32-bit lisp if you can, just to be sure.)} + +@subsection Upgrade from Elephant 0.5 + +Follow the upgrade procedures outlined in Elephant release 0.6.0 to migrate your database +from 0.5 to 0.6. Then follow the above procedures for upgrading from an 0.6 database. + +@emph{(NOTE: It may not take much work to make 0.9 upgrade directly from 0.5 +However there are so few (none?) 0.5 users that it wasn't deemed worth +the work given that there's an upgrade path available.)} @node CL-SQL @comment node-name, next, previous, up --- /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/04/27 03:14:55 1.13 +++ /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/04/28 02:31:07 1.14 @@ -140,13 +140,15 @@ Persistent collections inherit from @ref{Class elephant:persistent-collection} and consist of the @ref{Class elephant:btree}, @ref{Class elephant:indexed-btree} and @ref{Class elephant:btree-index} classes. The following operations are defined -on most of these classes. More information can be found in @ref{Using BTrees} -and @ref{Secondary Indices}. +on most of these classes. More information can be found in @ref{Persistent BTrees} +and @ref{BTree Indexing}. @include includes/fun-elephant-make-btree.texinfo - @include includes/fun-elephant-get-value.texinfo -@include includes/fun-elephant-setf-get-value.texinfo + +Values are written to a btree using the @code{setf} method on @code{get-value}. +@c @include includes/fun-elephant-setf-get-value.texinfo + @include includes/fun-elephant-remove-kv.texinfo @include includes/fun-elephant-existsp.texinfo @include includes/fun-elephant-drop-btree.texinfo @@ -178,7 +180,8 @@ @include includes/fun-elephant-make-cursor.texinfo @include includes/fun-elephant-cursor-close.texinfo @include includes/fun-elephant-cursor-duplicate.texinfo -@include includes/fun-elephant-cursor-initialized-p.texinfo +@c fixme, not generated by docstrings +@c @include includes/fun-elephant-cursor-initialized-p.texinfo Each of the following methods return multiple values consisting of @code{(exists? key value)}. @@ -197,8 +200,8 @@ @node Index Cursor API @comment node-name, next, previous, up -@section Cursor API -@cindex Cursors +@section Index Cursor API +@cindex Cursor @cindex Index @cindex Indices @cindex API @@ -224,12 +227,15 @@ @include includes/fun-elephant-cursor-pget-both.texinfo @include includes/fun-elephant-cursor-pget-both-range.texinfo -@include includes/fun-elephant-cursor-next-dup.texinfo @include includes/fun-elephant-cursor-next-nodup.texinfo -@include includes/fun-elephant-cursor-pnext-dup.texinfo +@include includes/fun-elephant-cursor-next-dup.texinfo @include includes/fun-elephant-cursor-pnext-nodup.texinfo +@include includes/fun-elephant-cursor-pnext-dup.texinfo + @include includes/fun-elephant-cursor-prev-nodup.texinfo +@include includes/fun-elephant-cursor-prev-dup.texinfo @include includes/fun-elephant-cursor-pprev-nodup.texinfo +@include includes/fun-elephant-cursor-pprev-dup.texinfo @node Transaction API @comment node-name, next, previous, up --- /project/elephant/cvsroot/elephant/doc/scenarios.texinfo 2007/04/27 03:14:55 1.9 +++ /project/elephant/cvsroot/elephant/doc/scenarios.texinfo 2007/04/28 02:31:07 1.10 @@ -711,14 +711,15 @@ hash table indirections with a little macro: @lisp -(defmacro def-snapshot-wrapper - (accessor-name (source-classname target-classname hashname uid)) +(defmacro def-snapshot-wrapper (accessor-name + (source-classname target-classname hashname uid)) (with-gensysms (obj key ref) `(progn (defmethod ,accessorname :around ((,obj ,source-classname)) (let ((,key (call-next-method))) (when ,key (gethash ,key ,hashname)))) - (defmethod (setf ,accessorname) :around (,ref (,obj ,source-classname)) + (defmethod (setf ,accessorname) :around + (,ref (,obj ,source-classname)) (cond ((subtypep (type-of ,ref) ,target-classname) (let ((,key (find-object ,ref ,hashname))) (if ,key --- /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/04/27 03:14:55 1.19 +++ /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/04/28 02:31:11 1.20 @@ -272,7 +272,7 @@ @item @strong{Merge-conflicts in heavily multi-process/threaded situations}. This is the common read-modify-write problem in all databases. We will talk -more about this in the @ref{Transactions} section. +more about this in the @ref{Using Transactions} section. @end enumerate @@ -761,7 +761,9 @@ (get-instances-by-range 'friends 'name "Adam" "Devin") => (#<Adriana> #<Carlos>) -(get-instances-by-range 'friend 'birthday (encode-date '(1 1 1974)) (encode-date '(31 12 1984))) +(get-instances-by-range 'friend 'birthday + (encode-date '(1 1 1974)) + (encode-date '(31 12 1984))) => (#<Zaid> #<Adriana>) (mapc #'print-friend *) @@ -1207,12 +1209,12 @@ and iterating of duplicate or unique values. @item @strong{Using the Map Operators} Mapping operators can be very efficient if properly utilized. -@item @strong{Using Multiple Stores} Multiple store controllers can -be open simultaneously. However it does make the code more complex -and you need to be careful about how you use them to avoid crashes -and other unpleasant side effects. -@item @strong{Custom Transaction Architecture} You can implement your -own version of @code{with-transaction} using the +@item @strong{Using Multiple Stores} +Multiple store controllers can be open simultaneously. However it +does make the code more complex and you need to be careful about how +you use them to avoid crashes and other unpleasant side effects. +@item @strong{Custom Transaction Architecture} +You can implement your own version of @code{with-transaction} using the underlying controller methods for starting, aborting and committing transactions. You had better know what you are doing, however! @item @strong{Handling Errors and Conditions} --- /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/04/27 03:14:55 1.19 +++ /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/04/28 02:31:11 1.20 @@ -188,7 +188,7 @@ @item @strong{Merge-conflicts in heavily multi-process/threaded situations}. This is the common read-modify-write problem in all databases. We will talk -more about this in the @ref{Transactions} section. +more about this in the @ref{Transaction Details} section. @item @strong{Byte Ordering}. The primitive elements such as integers are written to disk in @@ -908,7 +908,7 @@ Once a cursor is properly initialized, it can be incremented or decremented, a simple constant-time operation on BTrees. -@code{cursor-next} and @{cursor-prev} move the cursor a single step +@code{cursor-next} and @code{cursor-prev} move the cursor a single step forward or back across the sorted key-value pairs. @code{cursor-next} moves in ascending order, @code{cursor-prev} in descending order. @@ -960,10 +960,11 @@ class indexing capabilities previously described. An index is created by using the @code{add-index} function. This -function takes the @code{indexed-btree} you wish to index, an -index-name for later retrieval, a key-form which dictates how -the index populates it's keys as a function of the main btree's -keys and values. +function takes the @code{indexed-btree} you wish to index, an symbolic +name for the index and a key-form which dictates how the index +populates it's keys as a function of the main btree's keys and values. +(It is a function of three arguments: the index itself, the key and +the value). A simple, contrived example is shown in the figure below: @@ -1018,23 +1019,37 @@ Operations that have the same behavior, but return primary btree values and keys are: -@table @samp +@multitable @columnfractions .3 .1 .35 +@headitem BTree Cursor Function @tab @tab Index Cursor Function + @item @code{cursor-first} -@code{cursor-pfirst} -@samp{@code{cursor-first}} +@tab => +@tab @code{cursor-pfirst} + @item @code{cursor-last } -@code{cursor-plast} +@tab => +@tab @code{cursor-plast} + @item @code{cursor-current } -@code{cursor-pcurrent} +@tab => +@tab @code{cursor-pcurrent} + @item @code{cursor-next } -@code{cursor-pnext} +@tab => +@tab @code{cursor-pnext} + @item @code{cursor-prev} -@code{cursor-pprev} +@tab => +@tab @code{cursor-pprev} + @item @code{cursor-set} -@code{cursor-pset} +@tab => +@tab @code{cursor-pset} + @item @code{cursor-set-range} -@code{cursor-set-prange} -@end table +@tab => +@tab @code{cursor-set-prange} +@end multitable The big difference between btree cursors and index cursors is that indices can have duplicate key values. This means we have to choose @@ -1042,100 +1057,217 @@ a duplicate segment. There are cursor operations for each: @itemize -@item Simple move. Standard btree operations plus @code{cursor-pnext} and @code{cursor-pprev}. -@item Move to next key value. @code{cursor-pnext-nodup} and @code{cursor-pprev-nodup}. -@item Move to next duplicate. +@item Simple move. Standard btree operations work plus @code{cursor-pnext} and @code{cursor-pprev} +@item Move to a different key value. @code{cursor-pnext-nodup} and @code{cursor-pprev-nodup} +@item Move to next duplicate key value. @code{cursor-pnext-dup} and @code{cursor-pprev-dup} @end itemize -@c FINISH +After incrementing through a set of duplicate items using a +@code{xxx-dup} function, the last next operation returns nil +indicating there are no more duplicates. The consequence of this is +that the cursor is now uninitialized (@code{cursor-initialized-p}) and +needs to be reset by a set or set both call. + + @node Multi-threaded Applications @comment node-name, next, previous, up @section Multi-threaded Applications -Berkeley DB plays well with threads and processes. The store -controller is thread-safe by default, that is, can be shared amongst -threads. This is enabled by the @code{:thread} keyword argument which -defaults to true. Transactions may not be shared amongst threads -except serially. One thing which is NOT thread and process safe is -recovery, which should be run when no one is else is talking to the -database environment. - -The following shared regions of Elephant are protected by standard locks: -- buffer stream pool -- access to serializer circular buffers -- writes to connection db -- where else? +Elephant is thread-safe by design. Users should not have to think +about threading except to follow a couple of simple rules. + +@enumerate +@item Do not perform transactions across multiple threads +@item Do not perform add/remove index operations on indexed-btrees +in more than one thread. +@end enumerate + +This and common coding sense should be sufficient! Elephant's +internal design for thread safety employs a number of policies +to try to minimize using lisp locks and simplify analysis of +multi threaded interactions: + +@enumerate +@item @strong{Rely on the thread safety of the data store databases} +@item @strong{Ensure transaction isolation} +@item @strong{Minimize dependency on thread-local special variables} +@item @strong{Protect shared resources for a given store controller}. +@item @strong{A use policy for shared objects (above)} +@end enumerate + +@subsection Shared Resources + +Elephant has a few shared resources which are protected by standard locks. +These are: +@itemize +@item The store controller connection table +@item The instance cache +@item The circularity buffer pool for the serializer +@item The buffer-stream pool in memutils +@end itemize + +In some cases, and on some lisp platforms, we try to use a fast lock +strategy for frequently accessed items (the resource pools and +instance cache especially). + +@subsection Data Store Thread Safety and Transactions + +Both CLSQL and Berkelely DB backends are thread safe. In CLSQL this +is by ensuring that every thread has it's own handle into the SQL +libraries or sockets. Berkeley DB is reentrant and handles locking +internally. + +Elephant depends on these guarantees especially for the isolation +properties of transactions. All operations in the context of a +given transaction should be isolated and atomic. It is important +that a transaction not be shared across threads, however. + +@subsection Minimize Dependency on Thread-Local Specials + +Elephant uses several global variables as default arguments. Most +of these were removed leaving only a couple to handle: + +@itemize +@item @strong{@code{*store-controller*}}. Store controller objects can +be shared between threads and if a user resets this variable in a local +thread to another controller, there is no problem with that either. +Users of multiple concurrent stores can specify the store controller to +all elephant API commands that don't get it from a persistent object +implicitely. +@code{*current-transaction*}. This is always set to the proper null +value globally and should not be reset in local threads. Instead, +transactions take place in a dynamic context that rebinds this variable +as a special with the current transaction. This allows for a dynamic +transaction stack for data stores that can nest transactions or when +two datastores are both doing transactions concurrently. +@end itemize -@c *** FINISH *** @node Transaction Details @comment node-name, next, previous, up @section Transaction Details +Transactions are dynamic contexts in which all side effects to +persistent slots and other persistent objects such as BTrees are +guaranteed to have the ACID properties: atomicity, consistency, +isolation and durability. On a normal exit from context, the +side effects are committed as a group. On a non-local exit, +the transaction is aborted. + +For most users, the tutorial section @ref{Using Transactions} is the +best introduction to transactions. This section adds to that by +exposing some of the details of how it is implemented. + +To reiterate, there are a few important restrictions to adhere to: + +@itemize +@item @code{*current-transaction*} is reserved for use by the transaction system. Users should not override, manipulate or close over this variable. +@item The body of a transaction cannot throw, signal or jump without aborting the transaction. Any non-local exit is considered an aborting event. Catch signals inside the transaction and return a value instead. +@item The dynamic extent of a transaction body must stay within the same thread +@end itemize + +@subsection @code{with-transaction} internals + +The @code{with-transaction} macro wraps the body expression with an +anonymous lambda expression. This closure is passed to a call to the +@code{execute-transaction} generic function which is specialized to +the current data store. + +The only bookkeeping done by the macro is ensuring that the +@code{:parent} argument is checked for the current dynamic transaction +context. If it is not owned by the default or provided store +controller, then it is not passed to @code{execute-transaction}. This +maintains a continuous dynamic stack transactions through the +with/ensure transaction macros, but allows for a single leaf +transaction to another store controller. + +Be very careful about mixing transactions between store controller. +This facility was only added to ensure that migrate worked correctly. + +The macro processes keywords arguments @code{:store-controller} +(defaults to @code{*store-controller*}), @code{:parent} (defaults to +@code{*current-transaction*}) and @code{:retries} and passes the +remaining keywords to the call to @code{execute-transaction} allowing +the user to pass data store specific transaction keywords to their +preferred data store. The consumed keywords are analyzed and then +passed on to @code{execute-transaction}. + +Any non-standard keywords for a given data store will be ignored by +other data store implementation of @code{execute-transaction} so +portable programs should not use keywords that change the semantics of +the transaction. + +@code{ensure-transaction} only calls @code{execute-transaction} if +it needs to create a fresh transaction. If the transaction in +@code{*current-transaction*} exists and belongs to the store controller +passed to @code{ensure-transaction} then it merely calls the transaction +closure, relying on the environment that created the transaction to +handle any exit procedures and determining whether to abort or commit. + +@code{*current-transaction*} contains transaction records during the +dynamic execution of a transaction. These records capture any data +store specific bookkeeping as well as the store-controller that the +transaction is associated with. + +@subsection @code{execute-transaction} internals + +See the @ref{Elephant Architecture} section for details on how +execute-transaction works. It will provide some deeper insight +into the transaction system. + +@subsection Building your own transactional framework + +Data stores are required to implement three primitive transaction +methods: @code{controller-start-transaction}, +@code{controller-abort-transaction} and +@code{controller-commit-transaction}. These are wrappers for the data +store's primitive transaction mechanism. If you use these, it is up +to you to make sure that you properly manage nested transactions, +maintain the state of @code{*current-transaction*} handle any +automated retries you might want, and handle detecting + +If you use these, you are on your own - it is easy to make mistakes with +transactions and create very complex bugs that are hard to track down. +Most users are much better off sticking with the two transaction macros +and the underlying @code{execute-transaction} method. + +@subsection Analyzing Dynamic Transaction Behavior + You can trace @code{elephant::execute-transaction} to see the sequence -of calls to @code{execute-transaction} that occur dynamically and -detect where transactions are and are not happening. We may add some -transaction diagnosis and tracing tools in the future, such as -throwing a condition when @code{with-transaction} forms are nested -dynamically. - -;; Transaction architecture: -;; -;; User and designer considerations: -;; - *current-transaction* is reserved for use by dynamic transaction context. The default global -;; value must always be null (no transaction). Each data storeend can set it to a different parameter -;; within the dynamic context of an execute-transaction. -;; - Any closures returned from within a transaction cannot bind *current-transaction* -;; - Only a normal return value will result in the transaction being committed, any non-local exit -;; results in a transaction abort. If you want to do something more sophisticated, roll your own -;; using controller-start-transaction, etc. -;; - The body of a with or ensure transaction can take any action (throw, signal, error, etc) -;; knowing that the transaction will be aborted -;; +of calls that occur dynamically and detect where and how many +transactions are and are not happening. + +@c IT WOULD BE GOOD TO EXPAND ON THIS REGARDING HOW TO SOLVE COMMON PROBLEMS -@c *** FINISH *** @node Multi-repository Operation @comment node-name, next, previous, up @section Multi-repository Operation -Elephant now keeps a small hashtables that maps ``database specifications'' into -actual database connections. +Elephant maintains a small hashtable that maps ``database +specifications'' into actual @code{store-controller} objects. + +The basic strategy is that the ``database specification'' object is +stored in every persistent object and collection so that the +repository can be found. In this way, objects that reside in +different repositories can coexist within the LISP object space, +allowing data migration or multiple user stores. + +All persistent instances store their oid and a store-controller +reference in internal slots. Slot access and other protocols use +this to provide access. This executes an auto-transaction or joins +a surrounding transaction if the @code{transaction-record} in +@code{*current-transaction*} matches the store. + +When operating with multiple stores and nested transactions there are +some subtle issues to work around: how to avoid writing one store with +a transaction created in the context of another. A nested or ensured +transaction is only indicated in the call to +@code{execute-transaction} if the store controllers match, otherwise a +new transaction for that store is created. + +@c A PICTURE OF THE DYNAMIC CONTEXT WOULD BE USEFUL HERE -If a database spec is a string, it is assumed to be a BerkeleyDB path. -If it is a list, it is a assumed to be a CLSQL connection specification. -For example: -@lisp -ELE-TESTS> *testdb-path* -"/home/read/projects/elephant/elephant/tests/testdb/" -ELE-TESTS> *testpg-path* -(:postgresql "localhost.localdomain" "test" "postgres" "") -ELE-TESTS> -@end lisp -The tests now have a function @code{do-all-tests-spec} that take a spec and -based on its type attempt to open the correct kind of store controller and -perform the tests. - -The routine @code{get-controller} takes this specifiation. - -The basic strategy is that the ``database specification'' object is stored in -every persistent object and collection so that the repository can be found. - -In this way, objects that reside in different repositories can coexist within -the LISP object space, allowing data migration. - -;; Multiple stores - -;; Multiple store considerations: -;; - When operating with multiple stores, nested transactions and BDB there are some subtle issues to -;; work around: how to avoid writing one store with a transaction created in the context of another. -;; - For many leaf functions: *store-controller* and *current-transaction* have to both be correct; -;; this requirement may relax in the future -;; - The following macros accomodate multiple stores by requiring that execute-transaction return a -;; pair of (store-controller . txn-obj) where txn-obj is owned by the backend and the store-controller -;; is the store instance it is associated with. A nested or ensured transaction is only indicated -;; in the call to execute transaction if the store controllers match, otherwise a new transaction -;; for that store is created @node Multiple Processes and Distributed Applications @comment node-name, next, previous, up

1 0

[elephant-cvs] CVS elephant
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv16753 Modified Files: CREDITS README Removed Files: BEN-RETIRING-TODO.txt ChangeLog INSTALL NEWS NOTES TODO TUTORIAL UPGRADE-BDB Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/CREDITS 2006/11/11 06:27:37 1.8 +++ /project/elephant/cvsroot/elephant/CREDITS 2007/04/28 02:31:06 1.9 @@ -57,3 +57,16 @@ Marco Baringer provided a partial patch for 64-bit support. +Pierre Thierry for a *features* hack to turn off optimization as well as +many suggestions and dicussions. + +Henrik Hjelte for a number of bug fixes and for testing as well as +his wonderful postmodern data store! + +Edi Weisz for Lispwork and Win32 patches + +Mac Chan for his Hutchentoot blog tutorial + +And many others for their comments, testing and suggestions -- we're sorry +if we missed crediting you here. + --- /project/elephant/cvsroot/elephant/README 2006/11/11 06:27:37 1.6 +++ /project/elephant/cvsroot/elephant/README 2007/04/28 02:31:06 1.7 @@ -1,89 +1,58 @@ -============================================== -Elephant -- an object database for Common Lisp -============================================== +================================================================================= +Elephant -- An persistent metaobject protocol and object database for Common Lisp +================================================================================= ------------- -Introduction ------------- - -Elephant is an object database for Common Lisp. It supports -storing CLOS objects and most lisp primitives, and access to -BTrees. It uses Sleepycat / Berkeley DB, a -widely-distributed embedded database; many unix systems have -it installed by default. Sleepycat is server-less, ACID -compliant, transactional, process and thread safe, and fast -relative to relational databases; hopefully Elephant -inherits these properties. - -Goals: - -* Transparency: most Lisp values are easy to persist without -much effort or special syntax. Talk to the DB with Lisp -code, not SQL or another domain-specific language. No -additional server to run. - -* Safety: ACID, transactions. Concurrent with good -multi-user and -thread semantics / isolation, locking and -deadlock detection. - -* Simplicity: be a small library with few surprises for the -programmer. Lisp and Berkeley DB together are an excellent -substrate, try to use their features as much as possible. - -* Performance: leverage Sleepycat performance and -reliability. In addition to fast concurrent / transactional -modes, elephant will (eventually) offer an accellerated -single-user mode. - -Join the Elephant mailing lists to ask your questions and -receive updates. They're on the Elephant website +Detailed information on Elephant with an introduction, supported platforms, +latest news and documentation can be found at: http://www.common-lisp.net/project/elephant ------------------- -Supported Lisps ------------------- -CMUCL Linux (lightly tested) -SBCL Linux Mac <future?> (heavily tested) -Allegro Linux Mac cygwin? (heavily tested) -CLISP Linux Mac cygwin (lightly tested) -MCL Mac (lightly tested) - ----------------------------- Copyright, License + Warrenty ----------------------------- -See LICENSE. +See LICENSE + +------------------- +Supported Platforms +------------------- + +See the website: http://www.common-lisp.net/project/elephant/platforms.html ------------ -Installation +Latest News ------------ -See INSTALL +See the website: http://www.common-lisp.net/project/elephant/news.html ----------------------- -Tutorial and Reference ----------------------- +-------------- +Documentation +-------------- -HTML docs and texinfo sources can be found in the docs/ directory. +HTML and PDF documentation can be found on the website, or built via a +Makefile (and SBCL) in the /doc directory + +------------ +Installation +------------ + +See the documentation, Chapter 3 (Installation) ------------------------ Design and Development ------------------------ -See NOTES for some design internals. +See the documentation, Chapter 7 (Elephant Architecture) and +Chapter 8 (Data Store API) for some design internals. + +If you want to contribute, see the Trac website for current +feature release plans and other things that need to be done. -If you want to contribute, see TODO for current feature release plans -and other things that need to be done. +http://trac.common-lisp.net/elephant ------- Authors ------- -See CREDITS. - ----- -News ----- +See CREDITS and LICENSE -See NEWS and ChangeLog. \ No newline at end of file

1 0

[elephant-cvs] CVS elephant/doc/oldfiles
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/doc/oldfiles In directory clnet:/tmp/cvs-serv16495/oldfiles Log Message: Directory /project/elephant/cvsroot/elephant/doc/oldfiles added to the repository

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 27 Apr '07

27 Apr '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv32343/src/elephant Modified Files: classes.lisp collections.lisp package.lisp query.lisp transactions.lisp Log Message: Bug fixes to change-class; drop-btree; enable :from-end and :collect on map-btree (not map-class though); export and documentation edits --- /project/elephant/cvsroot/elephant/src/elephant/classes.lisp 2007/04/25 02:28:01 1.32 +++ /project/elephant/cvsroot/elephant/src/elephant/classes.lisp 2007/04/27 13:32:16 1.33 @@ -185,13 +185,17 @@ ;; CLASS CHANGE PROTOCOL ;; -(defmethod change-class ((inst persistent) (class t) &rest rest) - (cerror "Ignore and continue?" - "Changing a persistent instance's class to a non-persistent class is not currently allowed")) - -(defmethod change-class ((inst standard-object) (class persistent-metaclass) &rest rest) - (cerror "Ignore and continue?" - "Changing a standard instance to a persistent instance is not supported")) +(defmethod change-class :around ((previous persistent) (new-class standard-class) &rest initargs) + (declare (ignorable initargs)) + (unless (subtypep (type-of new-class) 'persistent-metaclass) + (error "Persistent instances cannot be changed to non-persistent classes in change-class")) + (call-next-method)) + +(defmethod change-class :around ((previous standard-object) (new-class persistent-metaclass) &rest initargs) + (declare (ignorable initargs)) + (unless (subtypep (type-of previous) 'persistent) + (error "Standard classes cannot be changed to non-persistent classes in change-class")) + (call-next-method)) (defmethod update-instance-for-different-class :around ((previous persistent) (current persistent) &rest initargs &key) (let* ((old-class (class-of previous)) --- /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/25 02:28:01 1.26 +++ /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/27 13:32:17 1.27 @@ -70,6 +70,18 @@ (defmethod optimize-layout ((bt t) &key &allow-other-keys) t) +(defgeneric drop-btree (bt) + (:documentation "Delete all key-value pairs from the btree and + render it an invalid object in the data store")) + +(defmethod drop-btree ((bt btree)) + (ensure-transaction (:store-controller *store-controller*) + (with-btree-cursor (cur bt) + (loop for (exists? key) = (multiple-value-list (cursor-first cur)) + then (multiple-value-list (cursor-next cur)) + while exists? + do (remove-kv key bt))))) + ;; ;; Btrees that support secondary indices ;; @@ -161,17 +173,17 @@ (defclass cursor () ((oid :accessor cursor-oid :type fixnum :initarg :oid) -;; (intialized-p cursor) means that the cursor has -;; a legitimate position, not that any initialization -;; action has been taken. The implementors of this abstract class -;; should make sure that happens under the sheets... -;; According to my understanding, cursors are initialized -;; when you invoke an operation that sets them to something -;; (such as cursor-first), and are uninitialized if you -;; move them in such a way that they no longer have a legimtimate -;; value. (initialized-p :accessor cursor-initialized-p - :type boolean :initform nil :initarg :initialized-p) + :type boolean :initform nil :initarg :initialized-p + :documentation "Predicate indicating whether +the btree in question is initialized or not. Initialized means +that the cursor has a legitimate position, not that any +initialization action has been taken. The implementors of this +abstract class should make sure that happens under the +sheets... Cursors are initialized when you invoke an operation +that sets them to something (such as cursor-first), and are +uninitialized if you move them in such a way that they no longer +have a legimtimate value.") (btree :accessor cursor-btree :initarg :btree)) (:documentation "A cursor for traversing (primary) BTrees.")) @@ -240,13 +252,13 @@ (defgeneric cursor-delete (cursor) (:documentation - "Delete by cursor. The cursor is at an invalid position -after a successful delete.")) + "Delete by cursor. The cursor is at an invalid position, +and uninitialized, after a successful delete.")) (defgeneric cursor-put (cursor value &key key) (:documentation - "Put by cursor. Currently doesn't properly move the -cursor.")) + "Overwrite value at current cursor location. Currently does + not properly move the cursor.")) (defclass secondary-cursor (cursor) () (:documentation "Cursor for traversing secondary indices.")) @@ -354,7 +366,7 @@ (defun lisp-compare-equal (a b) (equal a b)) -(defgeneric map-btree (fn btree &rest args &key start end value &allow-other-keys) +(defgeneric map-btree (fn btree &rest args &key start end value from-end collect &allow-other-keys) (:documentation "Map btree maps over a btree from the value start to the value of end. If values are not provided, then it maps over all values. BTrees do not have duplicates, but map-btree can also be used with indices @@ -365,26 +377,47 @@ ;; function orders by type tag and nil is the highest valued type tag so nils are the last ;; possible element in a btree ordered by value. -(defmethod map-btree (fn (btree btree) &rest args &key start end (value nil value-set-p) &allow-other-keys) - (let ((end (if value-set-p value end))) +(defmethod map-btree (fn (btree btree) &rest args &key start end (value nil value-set-p) + from-end collect &allow-other-keys) + (let ((end (if value-set-p value end)) + (results nil)) (ensure-transaction (:store-controller (get-con btree) :degree-2 *map-using-degree2*) (with-btree-cursor (curs btree) - (multiple-value-bind (exists? key value) - (cond (value-set-p - (cursor-set curs value)) - ((null start) - (cursor-first curs)) - (t (cursor-set-range curs start))) - (if exists? - (funcall fn key value) - (return-from map-btree nil)) - (loop - (multiple-value-bind (exists? k v) - (cursor-next curs) - (declare (dynamic-extent exists? k v)) - (if (and exists? (or (null end) (lisp-compare<= k end))) - (funcall fn k v) - (return nil))))))))) + (flet ((continue-p (key) + ;; Do we go to the next value? + (or (if from-end (null start) (null end)) + (if from-end + (or (not (lisp-compare<= key start)) + (lisp-compare-equal key start)) + (lisp-compare<= key end)))) + (collector (k v) + (push (funcall fn k v) results))) + (let ((fn (if collect #'collector fn))) + (declare (dynamic-extent (function continue-p) (function collector))) + (multiple-value-bind (exists? key value) + (cond (value-set-p + (cursor-set curs value)) + ((and (not from-end) (null start)) + (cursor-first curs)) + ((and from-end (null end)) + (cursor-last curs)) + (t (if from-end + (cursor-set-range curs end) + (cursor-set-range curs start)))) + (declare (dynamic-extent exists? k v)) + (if exists? + (funcall fn key value) + (return-from map-btree nil)) + (loop + (multiple-value-bind (exists? k v) + (if from-end + (cursor-prev curs) + (cursor-next curs)) + (declare (dynamic-extent exists? k v)) + (if (and exists? (continue-p k)) + (funcall fn k v) + (return nil))))))))) + results)) (defgeneric map-index (fn index &rest args &key start end value from-end collect &allow-other-keys) (:documentation "Map-index is like map-btree but for secondary indices, it --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/25 02:28:02 1.33 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/27 13:32:17 1.34 @@ -216,12 +216,11 @@ #:btree-index #:add-index #:get-index #:remove-index #:map-indices #:get-primary-key #:primary #:key-form #:key-fn - #:with-btree-cursor #:map-btree #:map-index + #:with-btree-cursor #:map-btree #:map-index #:drop-btree #:empty-btree-p #:dump-btree #:btree-keys #:btree-differ-p #:cursor #:secondary-cursor #:make-cursor #:make-simple-cursor - #:cursor-close #:cursor-init - #:cursor-duplicate #:cursor-current #:cursor-first + #:cursor-close #:cursor-duplicate #:cursor-current #:cursor-first #:cursor-last #:cursor-next #:cursor-next-dup #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup #:cursor-set #:cursor-set-range #:cursor-get-both @@ -229,8 +228,8 @@ #:cursor-pcurrent #:cursor-pfirst #:cursor-plast #:cursor-pnext #:cursor-pnext-dup #:cursor-pnext-nodup #:cursor-pprev #:cursor-pprev-nodup #:cursor-pset - #:cursor-pset-range #:cursor-pget-both - #:cursor-pget-both-range + #:cursor-pset-range #:cursor-pget-both #:cursor-pget-both-range + #:cursor-initialized-p #:find-class-index #:find-inverted-index #:enable-class-indexing #:disable-class-indexing @@ -240,7 +239,7 @@ #:report-indexed-classes #:class-indexedp-by-name - #:map-class #:map-class-index + #:map-class #:map-inverted-index #:get-instances-by-class #:get-instance-by-value #:get-instances-by-value --- /project/elephant/cvsroot/elephant/src/elephant/query.lisp 2007/04/12 02:47:33 1.4 +++ /project/elephant/cvsroot/elephant/src/elephant/query.lisp 2007/04/27 13:32:17 1.5 @@ -73,8 +73,8 @@ (if (find-inverted-index class slot) (if (= (length values) 1) (progn - (map-class-index fn class slot (first values) (first values)) - (map-class-index fn class slot (first values) (second values)))) + (map-inverted-index fn class slot (first values) (first values)) + (map-inverted-index fn class slot (first values) (second values)))) (map-class #'filter-by-relation class)) (map-class-query #'filter-by-relation (cdr constraints)))))) --- /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/12 02:47:33 1.11 +++ /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/27 13:32:17 1.12 @@ -95,13 +95,18 @@ ;; form)) (defun transaction-object-p (txnrec) - (consp txnrec)) + (and (not (null txnrec)) + (consp txnrec) + (subtypep (type-of (car txnrec)) 'store-controller))) (defun owned-txn-p (sc parent-txn-rec) (and parent-txn-rec (transaction-object-p parent-txn-rec) (eq sc (transaction-store parent-txn-rec)))) +(define-condition transaction-retry-count-exceeded () + ((retry-count :initarg :count))) + (defmacro with-transaction ((&rest keyargs &key (store-controller '*store-controller*) (parent '*current-transaction*) @@ -126,7 +131,7 @@ (defmacro ensure-transaction ((&rest keyargs &key (store-controller '*store-controller*) - (transaction '*current-transaction*) + (parent '*current-transaction*) (retries 200) &allow-other-keys) &body body) @@ -139,7 +144,7 @@ (sc (gensym))) `(let ((,txn-fn (lambda () ,@body)) (,sc ,store-controller)) - (if (owned-txn-p ,sc ,transaction) + (if (owned-txn-p ,sc ,parent) (funcall ,txn-fn) (funcall #'execute-transaction ,store-controller ,txn-fn

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 27 Apr '07

27 Apr '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv32343/src/db-bdb Modified Files: bdb-transactions.lisp Log Message: Bug fixes to change-class; drop-btree; enable :from-end and :collect on map-btree (not map-class though); export and documentation edits --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/25 02:27:58 1.14 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/27 13:32:14 1.15 @@ -57,7 +57,12 @@ (db-transaction-abort txn))))) (when success (return (values-list result))))) - finally (error "Too many retries in transaction")))) + finally (cerror "Retry transaction again?" + 'transaction-retry-count-exceeded + :format-control "Transaction exceeded the ~A retries limit" + :format-arguments (list retries) + :count retries)))) + (defmethod controller-start-transaction ((sc bdb-store-controller) &key

1 0