- elephant-cvs@lists.common-lisp.net - mailman3.common-lisp.net

[elephant-cvs] CVS elephant/src/elephant
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv16753/src/elephant Modified Files: classindex.lisp collections.lisp controller.lisp package.lisp transactions.lisp Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/04/24 12:58:10 1.39 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/04/28 02:31:15 1.40 @@ -84,11 +84,23 @@ (let ((class (find-class class-name nil))) (when class (indexed class)))) +(define-condition persistent-class-not-indexed (error) + ((class-obj :initarg :class :initarg nil :reader unindexed-class-obj))) + +(defun signal-class-not-indexed (class) + (cerror "Ignore and continue?" + 'persistent-class-not-indexed + :format-control "Class ~A is not enabled for indexing" + :format-arguments (list (class-name class)) + :class class)) + +;; (define-condition + (defmethod find-class-index ((class persistent-metaclass) &key (sc *store-controller*) (errorp t)) (ensure-finalized class) (if (not (indexed class)) (when errorp - (error "Class ~A is not an indexed class" class)) + (signal-class-not-indexed class)) (if (class-index-cached? class) (%index-cache class) ;; we've got a cached reference, just return it (multiple-value-bind (btree found) @@ -110,31 +122,26 @@ (synchronize-class-to-store class :sc sc :method method) btree)) -(define-condition persistent-class-not-indexed (error) - ((class-obj :initarg :class :initarg nil :reader unindexed-class-obj))) - (defun cache-new-class-index (class sc) "If not cached or persistent then this is a new class, make the new index" (if (indexed class) (enable-class-indexing class (indexing-record-slots (indexed-record class)) :sc sc) - (signal 'persistent-class-not-indexed - :class class - :format-control "Class ~A is not enabled for indexing" - :format-arguments (list (class-name class))))) + (signal-class-not-indexed class))) (defmethod find-inverted-index ((class symbol) slot &key (null-on-fail nil)) (find-inverted-index (find-class class) slot :null-on-fail null-on-fail)) (defmethod find-inverted-index ((class persistent-metaclass) slot &key (null-on-fail nil)) - (let* ((cidx (find-class-index class)) + (let* ((cidx (find-class-index class :errorp (not null-on-fail))) (idx (or (get-index cidx slot) (get-index cidx (make-derived-name slot))))) (if idx idx (if null-on-fail nil - (error "Inverted index ~A not found for class ~A with - persistent slots: ~A" slot (class-name class) (car (%persistent-slots class))))))) + (cerror "Ignore and continue?" + "Inverted index ~A not found for class ~A with persistent slots: ~A" + slot (class-name class) (car (%persistent-slots class))))))) (defmethod find-inverted-index-names ((class persistent-metaclass)) (let ((names nil)) --- /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/27 13:32:17 1.27 +++ /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/28 02:31:15 1.28 @@ -312,6 +312,7 @@ primary key greater or equal to the pkey argument. Returns has-tuple / secondary key / value / primary key.")) + (defgeneric cursor-next-dup (cursor) (:documentation "Move to the next duplicate element (with the same key.) @@ -322,11 +323,6 @@ "Move to the next non-duplicate element (with different key.) Returns has-pair key value.")) -(defgeneric cursor-prev-nodup (cursor) - (:documentation - "Move to the previous non-duplicate element (with -different key.) Returns has-pair key value.")) - (defgeneric cursor-pnext-dup (cursor) (:documentation "Move to the next duplicate element (with the same key.) @@ -338,12 +334,53 @@ key.) Returns has-tuple / secondary key / value / primary key.")) + +(defgeneric cursor-prev-dup (cursor) + (:documentation + "Move to the previous duplicate element (with the same key.) +Returns has-pair key value.")) + +(defmethod cursor-prev-dup ((cur cursor)) + "Default implementation. Plan is to update both backends when BDB 4.6 comes out" + (when (cursor-initialized-p cur) + (multiple-value-bind (exists? skey-cur) + (cursor-current cur) + (declare (ignore exists?)) + (multiple-value-bind (exists? skey value) + (cursor-prev cur) + (if (lisp-compare-equal skey-cur skey) + (values exists? skey value) + (setf (cursor-initialized-p cur) nil)))))) + +(defgeneric cursor-prev-nodup (cursor) + (:documentation + "Move to the previous non-duplicate element (with +different key.) Returns has-pair key value.")) + +(defgeneric cursor-pprev-dup (cursor) + (:documentation + "Move to the previous duplicate element (with the same key.) +Returns has-tuple / secondary key / value / primary key.")) + +(defmethod cursor-pprev-dup ((cur cursor)) + "Default implementation. Plan is to update both backends when BDB 4.6 comes out" + (when (cursor-initialized-p cur) + (multiple-value-bind (exists? skey-cur) + (cursor-current cur) + (declare (ignore exists?)) + (multiple-value-bind (exists? skey value pkey) + (cursor-pprev cur) + (if (lisp-compare-equal skey-cur skey) + (values exists? skey value pkey) + (setf (cursor-initialized-p cur) nil)))))) + (defgeneric cursor-pprev-nodup (cursor) (:documentation "Move to the previous non-duplicate element (with different key.) Returns has-tuple / secondary key / value / primary key.")) + (defmacro with-btree-cursor ((var bt) &body body) "Macro which opens a named cursor on a BTree (primary or not), evaluates the forms, then closes the cursor." @@ -439,6 +476,7 @@ start end)) (let ((sc (get-con index)) (end (or value end)) + (from-end (and from-end (not value-set-p))) (results nil)) (flet ((collector (k v pk) (push (funcall fn k v pk) results))) @@ -454,11 +492,12 @@ (lisp-compare-equal key start)) (lisp-compare<= key end)))) (value-increment () - ;; Step to the next key value + ;; Step to the next key value; + ;; from-end duplicate cursor is already there (if from-end - (pprev-hack cur) + (cursor-current cur) (cursor-pnext-nodup cur))) - (next-value () + (map-values () ;; Handle the next key value (multiple-value-bind (exists? skey val pkey) (value-increment) @@ -468,18 +507,23 @@ (map-duplicates skey)) (return-from map-index (nreverse results))))) + (next-duplicate (key) + (if from-end + (pprev-dup-hack cur key) + (cursor-pnext-dup cur))) (map-duplicates (key) ;; Map all duplicates for key value (multiple-value-bind (exists? skey val pkey) - (cursor-pnext-dup cur) + (next-duplicate key) (if exists? (progn (funcall fn skey val pkey) (map-duplicates key)) (progn - (cursor-pset-range cur key) - (next-value)))))) - (declare (dynamic-extent (function next-value) (function next-value-increment) + (unless from-end + (cursor-pset cur key)) + (map-values)))))) + (declare (dynamic-extent (function map-values) (function next-duplicate) (function continue-p) (function map-duplicates))) (multiple-value-bind (exists? skey val pkey) (cond (value-set-p @@ -487,9 +531,9 @@ ((and (not from-end) (null start)) (cursor-pfirst cur)) ((and from-end (null end)) - (cursor-last-range-hack cur)) + (cursor-plast cur)) (t (if from-end - (cursor-pset-range cur end) + (pset-range-for-descending cur end) (cursor-pset-range cur start)))) (if (and exists? (continue-p skey)) (progn @@ -497,23 +541,24 @@ (map-duplicates skey)) nil))))))))) -(defun pprev-hack (cur) - "Get the first duplicate instance of the prior value off the current cursor" - (let ((e? (cursor-pprev-nodup cur))) - (when e? - (let ((e? (cursor-pprev-nodup cur))) - (if e? - (cursor-pnext cur) - (cursor-pfirst cur)))))) - -(defun cursor-last-range-hack (cur) - "Get the first duplicate instance of the last value of the cursor's index" - (let ((e? (cursor-plast cur))) - (when e? - (let ((e? (cursor-pprev-nodup cur))) - (if e? - (cursor-pnext cur) - (cursor-pfirst cur)))))) +(defun pset-range-for-descending (cur end) + (if (cursor-pset cur end) + (progn + (cursor-next-nodup cur) + (cursor-pprev cur)) + (progn + (cursor-pset-range cur end) + (cursor-pprev cur)))) + +(defun pprev-dup-hack (cur key) + "Go back one step in a duplicate set, returns nil + if previous element is a different key. More efficient than + the current default implementation of cursor-pprev-dup" + (multiple-value-bind (exists? skey value pkey) + (cursor-pprev cur) + (when (lisp-compare-equal key skey) + (values exists? key value pkey)))) + ;; =============================== ;; Some generic utility functions --- /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/04/25 02:28:01 1.50 +++ /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/04/28 02:31:15 1.51 @@ -136,7 +136,7 @@ (defun initialize-user-parameters () (loop for (keyword variable) in *user-configurable-parameters* do (awhen (get-user-configuration-parameter keyword) - (setf variable it)))) + (setq variable it)))) ;; ;; COMMON STORE CONTROLLER FUNCTIONALITY @@ -165,7 +165,7 @@ "This is an instance cache and part of the metaclass protocol. Data stores should not override the default behavior.") - (instance-cache-lock :accessor instance-cache-lock :initform (ele-make-lock) + (instance-cache-lock :accessor instance-cache-lock :initform (ele-make-fast-lock) :documentation "Protection for updates to the cache from multiple threads. Do not override.") @@ -202,14 +202,16 @@ (defun cache-instance (sc obj) "Cache a persistent object with the controller." (declare (type store-controller sc)) - (ele-with-lock ((instance-cache-lock sc)) + (ele-with-fast-lock ((instance-cache-lock sc)) (setf (get-cache (oid obj) (instance-cache sc)) obj))) (defun get-cached-instance (sc oid class-name) "Get a cached instance, or instantiate!" (declare (type store-controller sc) (type fixnum oid)) - (let ((obj (get-cache oid (instance-cache sc)))) + (let ((obj + (ele-with-fast-lock ((instance-cache-lock sc)) + (get-cache oid (instance-cache sc))))) (if obj obj ;; Should get cached since make-instance calls cache-instance (make-instance class-name :from-oid oid :sc sc)))) --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/27 13:32:17 1.34 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/28 02:31:16 1.35 @@ -222,12 +222,12 @@ #:cursor #:secondary-cursor #:make-cursor #:make-simple-cursor #:cursor-close #:cursor-duplicate #:cursor-current #:cursor-first #:cursor-last #:cursor-next #:cursor-next-dup - #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup + #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup #:cursor-prev-dup #:cursor-set #:cursor-set-range #:cursor-get-both #:cursor-get-both-range #:cursor-delete #:cursor-put #:cursor-pcurrent #:cursor-pfirst #:cursor-plast #:cursor-pnext #:cursor-pnext-dup #:cursor-pnext-nodup - #:cursor-pprev #:cursor-pprev-nodup #:cursor-pset + #:cursor-pprev #:cursor-pprev-dup #:cursor-pprev-nodup #:cursor-pset #:cursor-pset-range #:cursor-pget-both #:cursor-pget-both-range #:cursor-initialized-p @@ -267,6 +267,7 @@ ;; Various error conditions #:cross-reference-error #:controller-lost-error + #:persistent-class-not-indexed #:map-class-query #:get-query-instances --- /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/27 13:32:17 1.12 +++ /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/28 02:31:16 1.13 @@ -120,7 +120,7 @@ If nested, the backend must support nested transactions." (let ((sc (gensym))) `(let ((,sc ,store-controller)) - (funcall #'execute-transaction ,store-controller + (funcall #'execute-transaction ,sc (lambda () ,@body) :parent (if (owned-txn-p ,sc ,parent) (transaction-object ,parent) @@ -146,10 +146,9 @@ (,sc ,store-controller)) (if (owned-txn-p ,sc ,parent) (funcall ,txn-fn) - (funcall #'execute-transaction ,store-controller + (funcall #'execute-transaction ,sc ,txn-fn :parent nil - :transaction nil :retries ,retries ,@(remove-keywords '(:store-controller :parent :transaction :retries) keyargs))))))

1 0

[elephant-cvs] CVS elephant/doc/oldfiles
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/doc/oldfiles In directory clnet:/tmp/cvs-serv16753/doc/oldfiles Added Files: INSTALL NEWS NOTES TODO TUTORIAL Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/doc/oldfiles/INSTALL 2007/04/28 02:31:14 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/INSTALL 2007/04/28 02:31:14 1.1 ------------ Requirements ------------ Supported Lisps: CMUCL 19a Linux SBCL 0.9.17/1.0+ Linux / Mac OSX Allegro CL 7.0/8.0 Linux / Mac OSX OpenMCL 0.14.2 LispWorks (port in-progress) Lisp libraries: ASDF - http://www.cliki.net/asdf UFFI 1.5.4+ - http://uffi.b9.com/ A Backend Database: 1) Oracle Berkeley DB 4.5 - http://www.oracle.com/database/berkeley-db.html 2) CLSQL - http://clsql.b9.com/ with an appropriate SQL installation. Tested with SQlite3 and Postgresql so far A C compiler, probably gcc or Visual Studio. Presumably you have this if you installed ------------------ Short Instructions ------------------ The new build system should work out of the box on most Un*x platforms that have asdf, uffi and either clsql or Berkeley DB installed in the usual places. Try: (asdf:operate 'asdf:load-op :elephant) Then: (open-store '(<backend> <spec>)) Where <backend> = { :bdb | :clsql } <spec> = { "fresh directory for BDB files" | '(:sqlite3 "db path") | '(:postgresql "db path") This should load all files, including compiling libraries, on most systems. For Win32, see the instructions below. (We'll improve the build process for Win32 if there is demand) ----------------- Long Instructions ----------------- For SBCL, CMUCL, Allegro 8.0+, MCL and CLISP: 0) Unpack Elephant. I put mine in the directory /usr/local/share/common-lisp/elephant-0.6.x/ 1) Install ASDF. Ensure that you have a recent version of ASDF installed as the load process now depends upon it. 2) Install UFFI 3) Install a backend: Either Berkeley DB 4.5, PostGresql, or SQLite 3. ------- SQL ------- For relational database systems, refering the formal documentation other the heading "SQL-BACK-END". ------------- Berkeley 4.5: ------------- (Note: 0.6.0 required BDB 4.3; to upgrade 0.6.0 to 0.6.1, upgrade BDB to 4.5, modify my-config.sexp appropriately then run 0.6.1+; your underlying Berekely DB files will automatically upgrade when the DB is opened. To use 0.6.1, you will have to manually migrate your 0.6.0 database to a fresh database created in 0.6.1) Under Un*x, you may actually already have this installed, though it may be compiled with funny options, so if things don't work you may want to try to start from scratch. FreeBSD has a port for this, as I'm sure do other BSDs (including DarwinPorts/Fink.) Take note of where libdb.so and db.h are installed, usually: /usr/local/BerkeleyDB.4.5/lib/libdb.so and /usr/local/BerkeleyDB.4.5/include/db.h, or /usr/local/lib/db45/libdb.so and /usr/local/include/db45/db.h.) a) Site specific configuration config.sexp Which contains an alist providing string paths pointing to the root of the Berkeley DB distribution :berkeley-db-root, the library to load :berkeley-db-lib and the pthreads library if you're running linux :pthread-lib. For Win32 (directions courtesy of Bill Clementson): --------------------------------------------------- Create an MSVC dll project and add src/db-bdb/libberkeley-db.c, src/db-bdb/libberkeley-db.def and the Berkeley DB libdb43.lib files to the project (should be in the build_win32/release folder) Add the Berkeley DB dbinc include files directory and the build_win32/release directory (where the Berkeley DB install instructions builds the Berkeley DB objects by default) to the build directories for the project Build the Elephant DLL file Since you've statically included libdb43.lib inside libberkeley-db.c, it may or may not be necessary to load libdb43.dll into Lisp (see below.) 4) Compile and load Elephant: The new backend load process should work automatically on Un*x systems but if there are probolems with loading foreign libraries, then you can test your C tools setup with 'make' in the elephant root directory. This will build the common memutils library in src/memutil/libmemutil.so/dylib that all backends require. There is a new two-phase load process. The first requires that you use asdf to load the main elephant front-end: (asdf:operate 'asdf:load-op :elephant) This will load and compile Elephant. This will also automatically load UFFI. When you call (open-store <spec>) inside lisp it will automatically load the remaining dependencies for the specified backend via ASDF. To test the load process explicitly the following asdf files are provided: if you are using Berkeley DB, type: (asdf:operate 'asdf:load-op :ele-bdb) if you are using CL-SQL, type: (asdf:operate 'asdf:load-op :ele-clsql) if you are using SQLite3, type: (asdf:operate 'asdf:load-op :ele-sqlite3) 5) Make the documentation: Execute: make In the doc directory should be build the HTML version of the texinfo files. ------- Testing ------- Elephant uses RT for regression testing, available at: http://www.cliki.net/RT Once RT is installed (asdf:operate 'asdf:load-op :elephant-tests) (in-package :ele-tests) (setf *default-spec* <backend>) Where <backend> = { *testsqlite3-spec* | *testpg-spec* | *testbdb-spec* } (do-backend-tests) This will test the standalone API for your backend. Currently all tests are passing on 0.6.0. There will be a set of migration tests that will be 'ignored' but the final message should indicate no failing tests. This should take less than 5 minutes on decent hardware. The tests are not idempotent, so if you run the tests a second time, they are likely to fail. To avoid this, for example if you are debugging tests, just run the script delscript.sh (or do the equivalent on Win32) in the elephant/tests directory. Elephant allows migration between repositories. To test this: (do-migration-tests *default-spec* <backend>) where <backend> is a different *testXXXXX-spec* variable to test migration to that backend. This should take less than 2 minutes on decent hardware. A backend is considered "green" if it can pass both the backend tests and the migration tests. --- /project/elephant/cvsroot/elephant/doc/oldfiles/NEWS 2007/04/28 02:31:14 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/NEWS 2007/04/28 02:31:14 1.1 April, 2006 - Elephant 0.6.0 released by Robert Read and Ian Eslick. Supports class slot indexing and benefits from a clean refactoring of backends and a host of other small changes. This is a solid BETA release. November 30, 2005 - Elephant 0.3.0 released by the new maintainer, Robert L. Read, providing support for relational database backends, repository migration, and multi-repository operation. As of this release, the documentation provides a lot of information about installation and getting things working; I wouldn't at all claim that it is complete, smooth, or well organized. The more notes I get about the use of Elephant, the more inclined I will be to invest time in improving the documentation. October 7, 2004 - Elephant 0.2.1 released. Thanks to Bill Clementson, Elephant should compile on Win32 now. Also, a few minor fixups. September 19, 2004 - Elephant 0.2 released. This is an BETA release. New features: - Secondary indices and cursors - PPC Darwin OpenMCL / SBCL - Doc strings and improved documentation - An RT-based test suite - many bugfixes This release has been tested on CMUCL 19a, SBCL 0.8.14 and Allegro 6.2 on x86 Linux and FreeBSD, and OpenMCL 0.14.2-p1 and SBCL 0.8.14 on PPC Darwin. September 2, 2004 - The bad news: there was a bug in 0.1 which made OID generation inside of manual transactions deadlock. The good news: this is fixed, and I've added OpenMCL support. So I'm releasing 0.1-p1. August 30, 2004 - Elephant 0.1 was released August 30th, 2004. This is an ALPHA quality release, so claims about correctness, performance and safety should be taken with a grain of salt. This release has been tested on CMUCL 19a, SBCL 0.8.13 and Allegro 6.2 on x86 Linux and FreeBSD. OpenMCL and Lispworks versions will come soon. As a proof of concept I've compiled and run CL-IRC http:://www.common-lisp.net/project/cl-irc making all objects and slots persistent, except for the socket-streams. It runs, and saves everything except for the socket-streams. --- /project/elephant/cvsroot/elephant/doc/oldfiles/NOTES 2007/04/28 02:31:15 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/NOTES 2007/04/28 02:31:15 1.1 ------- GENERAL ------- this has been optimized for use with CMUCL / SBCL / Allegro. OpenMCL has been minimally supported. Lispworks is a target as well but less so as the developers don't have access to it. Theoretically one can port this to any lisp with a decent FFI and MOP. However since those are two of the less standardized bits of Lisp, in practice this might be difficult. >From top to bottom, here are the implementation layers: ELEPHANT package persistent meta-object, persistent collections controller serializer memutils package UFFI / implementation specific stuff libsleepycat.so Sleepycat 4.2/3 While I loath specials, since you can't change the signature of slot accessors, in order to pass parameters to the database / serializer, specials are needed. Also specials will probably play nice with threaded lisps. ----------------------- CLASSES AND METACLASSES ----------------------- Persistent classes which the user defines are declared and instrumented by using the persistent-metaclass. Ideally creating persistent versions of class, slot-defintion, et al would be enough, but in reality various implementations do things in different ways. CMUCL / SBCL: their's a bit of work to make class slot allocation and reader / writer / slot-boundp work right. Allegro: is using slot-boundp instead of slot-boundp-using-class inside of shared-initialize, which necessitates some work. CMUCL doesn't do non-standard allocation types correctly, so we've created our own slot definition keyword :transient. In the future this will change. Andrew will add some notes here in the future. ----------- COLLECTIONS ----------- While we support serializing and persisting a wide class of Lisp data types, there are problems with persisting aggregate types (conses, lists, arrays, objects, hash-tables...) 1) not automatic: there's no way for elephant to know when you've changed a value in an aggregate object, so you have to manually restore it back into the slot to get it saved. example 1: you put a cons into the database. you change it's car. this is not saved unless you resave the cons into the database. example 2: slot-1 of obj A (saved in the database) contains a cons. you change the car of the cons. this is not reflected into the database unless you resave A. 2) merge-conflicts: changing one value and saving an aggregate will write out the whole aggregate, possibly blowing away changes other threads have made behind your back. this is not protected by transactions! 3) consing, non-lazy and expensive (de)serialization: you have to serialize/deserialize the entire aggregate every time you save it or restore it. This is pretty fast all things considered, but it's probably better to use persistent collections. 4) you have to store the entire collection in memory, whereas one of the points of the database to store large collections of objects..... For these and other reasons, we provide a hash-table-like interface to Berkeley BTrees. These have many advantages over ordinary hash-tables from the point of view of persistence. There is a separate table for BTrees. This is because we use a hand coded C function for sorting, which understands a little of the serialized data. It can handle numbers (up to 64-bit bignums -- they are approximated by floats) and strings (case-insensitive for 8-bit, code-point-order for 16-bit Unicode.) It should be fast but we don't want a performance penalty on objects. Secondary indices are mostly handled on the lisp side, because of our weird table layout (see below) and to avoid crossing FFI boundaries. Some unscientific microbenchmarks indicated that there was no performance benefit on CMUCL / SBCL, and only minor benefit (asymptotically nil) on OpenMCL. They have a separate table. Actually two handles are opened on this table: one which is plain, and one which is associated to the primary btree table by a no-op indexing function. Since we maintain the secondary keys ourselves, the associated handle is good for gets / cursor traversals. We use the unassociated handle for updates. ---------- CONTROLLER ---------- The controller is accessed through the special *store-controller*. The controller keeps track of 1) the environment handle 2) the DB handle(s) 3) the instance cache 4) the root object The environment handle and DB handle currently aren't really exposed. Eventually they should be, so that tuning flags can be set on them. OIDs are generated by a bit of C code, which isn't great, nor that safe (to get acceptable performance i use DB_TXN_NOSYNC.) Waiting for Sleepycat 4.3. The instance cache is implemented as a values-weak hash-table. This is a hash-table where the values can be collected, and when they are, the entire key-value entry is deleted. There are implementations of this on the various platforms. The instance cache is there to make deserialization of persistant objects faster. Since we [249 lines skipped] --- /project/elephant/cvsroot/elephant/doc/oldfiles/TODO 2007/04/28 02:31:15 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/TODO 2007/04/28 02:31:15 1.1 [498 lines skipped] --- /project/elephant/cvsroot/elephant/doc/oldfiles/TUTORIAL 2007/04/28 02:31:15 NONE +++ /project/elephant/cvsroot/elephant/doc/oldfiles/TUTORIAL 2007/04/28 02:31:15 1.1 [923 lines skipped]

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv16753/src/db-bdb Modified Files: bdb-transactions.lisp Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/27 13:32:14 1.15 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/28 02:31:15 1.16 @@ -31,15 +31,13 @@ for count fixnum from 1 to retries for success of-type boolean = nil do - (let ((txn - (if transaction transaction - (db-transaction-begin env + (let ((txn (db-transaction-begin env :parent (if parent parent +NULL-VOID+) :degree-2 degree-2 :read-uncommitted read-uncommitted :txn-nosync txn-nosync :txn-nowait txn-nowait - :txn-sync txn-sync)))) + :txn-sync txn-sync))) (declare (type pointer-void txn)) (let (result) (let ((*current-transaction* (make-transaction-record sc txn))

1 0

[elephant-cvs] CVS elephant/doc
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/doc In directory clnet:/tmp/cvs-serv16753/doc Modified Files: copying.texinfo data-store-reference.texinfo elephant.texinfo installation.texinfo reference.texinfo scenarios.texinfo tutorial.texinfo user-guide.texinfo Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/doc/copying.texinfo 2007/03/30 14:34:34 1.4 +++ /project/elephant/cvsroot/elephant/doc/copying.texinfo 2007/04/28 02:31:07 1.5 @@ -8,29 +8,30 @@ @section Elephant Licensing -@b{Elephant}: a persistent metaprotocol and object-oriented database -for Common Lisp. +Elephant is a persistent metaprotocol and object-oriented database for +Common Lisp. Detailed information and distributions can be found at +@uref{http://www.common-lisp.net/project/elephant}. -Homepage: @uref{http://www.common-lisp.net/project/elephant} +The program is released under the following license: @quotation -Elephant users are granted the rights to distribute and use this software -as governed by the terms of the Lisp Lesser GNU Public License -@uref{http://opensource.franz.com/preamble.html}, also known as the LLGPL. +Elephant users are granted the rights to distribute and use this +software as governed by the terms of the Lisp Lesser GNU Public +License @uref{http://opensource.franz.com/preamble.html}, also known +as the LLGPL. @end quotation Copyrights include: @quotation -Copyright (c) 2004 by Andrew Blumberg and Ben Lee - -Copyright (c) 2005-2007 by Robert L. Read - -Copyright (c) 2006-2007 by Ian Eslick +Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. @* +Version 0.5, Copyright @copyright{} 2006 Robert L. Read. @* +Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* +Portions copyright respective contributors (see @file{CREDITS}). @end quotation -Portions of this program (namely the C unicode string sorter) are -derived from IBM's @b{ICU}: @uref{http://oss.software.ibm.com/icu/, +Portions of the program (namely the C unicode string sorter) are +derived from IBM's ICU: @uref{http://oss.software.ibm.com/icu/, ICU Website} whose copyright and license follows below. @quotation @@ -75,6 +76,33 @@ are the property of their respective owners. @end quotation +@section Elephant Manual Copyright and Licensing + +@quotation +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License. +@end quotation + +Copyrights include: + +@quotation +Original Version, Copyright @copyright{} 2004 Ben Lee. @* +Versions 0.5-0.6, Copyright @copyright{} 2006 Robert L. Read. @* +Current Version, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* +@end quotation + +@section 3rd Party Libraries + +Elephant depends on 3rd party lisp libraries. See their respective +distributions for detailed copyright and licensing information. The +following is a brief summary. + +@itemize +@item @b{uffi}: By Kevin Rosenberg, no significant restrictions +@item @b{cl-base64}: By Kevin Rosenberg, no significant restrictions +@item @b{rt}: By Richard Waters, MIT License +@end itemize + @section Data Store Licensing Considerations The Berkeley DB data store is based on the Berkeley DB C library, now @@ -100,5 +128,3 @@ @item MySQL: Dual licensing (similar to BDB), see @uref{http://www.mysql.com/company/legal/licensing/, the MySQL license page} @end itemize - - --- /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/04/27 03:14:55 1.7 +++ /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/04/28 02:31:07 1.8 @@ -34,7 +34,7 @@ * Cursors: DSR Cursors. Traversing BTrees. * Transactions: DSR Transactions. Transaction implementation. * Multithreading Considerations: DSR Multithreading Considerations. Multithreading considerations. -* Foreign Libraries: DSR Foreign Libraries. Using UFFI and ASDF to build or link foreign libraries +@c * Foreign Libraries: DSR Foreign Libraries. Using UFFI and ASDF to build or link foreign libraries @end menu @node DSR Registration @@ -147,8 +147,8 @@ implemented using map-btree and map-index. @itemize -@item @ref{Generic-Function elephant:get-value} -@item @ref{Generic-Function (setf elephant:get-value)} +@item @ref{Generic-Function elephant:get-value} (and @code{(setf get-value)}) +@c @item @ref{Generic-Function (setf elephant:get-value)} @item @ref{Generic-Function elephant:existsp} @item @ref{Generic-Function elephant:remove-kv} @item @ref{Generic-Function elephant:get-index} @@ -168,7 +168,7 @@ @cindex Cursors Data stores must subclass these cursor classes and implement all the -methods described in @ref{Cursors} except @ref{Macro +methods described in @ref{DSR Cursors} except @ref{Macro elephant:with-btree-cursor}. @include includes/class-elephant-cursor.texinfo @@ -220,9 +220,23 @@ @section Multithreading Considerations @cindex Multithreading -Generic locking utility functions +This expands slightly on the multithreading discussion in +@ref{Multi-threaded Applications}. -Variable behavior in multithreading situations +Elephant provides a set of generic locking functions in +(a)code{src/utils/locks.lisp} to help protect any shared structures. +There are standard locking functions (@code{ele-with-lock}) and then +a special locking interface called @code{ele-with-fast-lock} which +on some lisps provides a faster locking option than the standard OS +locks of the basic interface. (i.e. under Allegro this uses +@code{without-interrupts} because Allegro still runs in a single +OS process on all platforms, this is not true of SBCL). + +See the sections on Transaction handling, particularly the dynamic +behavior of @code{*current-transaction*}. Also read up on the +store controller section in the User Guide to better understand the +role of @code{*store-controller*}. At this time there are no other +global variables to worry about. @node DSR Handling Serialization @comment node-name, next, previous, up @@ -253,19 +267,19 @@ @include includes/fun-elephant-data-store-serialize-to-base64-string.texinfo @include includes/fun-elephant-data-store-deserialize-from-base64-string.texinfo -@node DSR Memory Utilities -@comment node-name, next, previous, up -@section Memory utilities -@cindex Memory utilities - -Details about memory utilities here. - -@node DSR Foreign Libraries -@comment node-name, next, previous, up -@section Foreign libraries -@cindex Foreign libraries +@c @node DSR Memory Utilities +@c @comment node-name, next, previous, up +@c @section Memory utilities +@c @cindex Memory utilities + +@c Details about memory utilities here. + +@c @node DSR Foreign Libraries +@c @comment node-name, next, previous, up +@c @section Foreign libraries +@c @cindex Foreign libraries -How foreign libraries are built and used via UFFI. What functions are -in the .asd files or main lisp code to build & load libraries? +@c How foreign libraries are built and used via UFFI. What functions are +@c in the .asd files or main lisp code to build & load libraries? --- /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/04/27 03:14:55 1.10 +++ /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/04/28 02:31:07 1.11 @@ -2,19 +2,27 @@ @c %**start of header @setfilename elephant.info @settitle Elephant User Manual +@setchapternewpage odd @c %**end of header - -@copying -Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. -Version 0.5, Copyright @copyright{} 2006 Robert L. Read. -Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read +@copying @quotation +Elephant System @* +Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. @* +Version 0.5, Copyright @copyright{} 2006 Robert L. Read. @* +Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* +Portions copyright respective contributors (see @file{CREDITS}). @* + +Elephant Manual @* +Original Version, Copyright @copyright{} 2004 Ben Lee. @* +Versions 0.5-0.6, Copyright @copyright{} 2006 Robert L. Read. @* +Current Version, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @* + Permission is granted to copy, distribute and/or modify this document -under the terms of the GNU Free Documentation License. Portions of -Elephant are derived from ICU, IBM's Unicode library. Details about -copyright, license and warranty information can be found in -@file{LICENSE} or in the ``Copying'' section of this document. +under the terms of the GNU Free Documentation License. See the +Copyright and License chapter for details about copyright, license and +warranty for this manual and the Elephant system. + @end quotation @end copying --- /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/04/27 03:14:55 1.11 +++ /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/04/28 02:31:07 1.12 @@ -9,9 +9,10 @@ * Requirements:: Supported lisps and required libraries. * Configuring Elephant:: Setting up Elephant and the configuration file. * Loading Elephant:: Loading Elephant and the data store loading protocol. -* Berkeley DB:: Installing support for the Berkeley DB data store +* Berkeley DB:: Installing support for the Berkeley DB data store. * Berkeley DB Example:: An example of installing and running the Berkeley DB data store. -* CL-SQL:: Install and connecting to the CL-SQL data store +* Upgrading Berkeley DB Database:: How to upgrade to a new version of Berkeley DB. +* CL-SQL:: Install and connecting to the CL-SQL data store. * CL-SQL Example:: An example of using the CL-SQL data store. * Elephant on Windows:: More details about running Elephant on Windows * Test Suites:: How to run and interpret the output of the regression test suite @@ -275,6 +276,57 @@ @end lisp in your application. +@node Upgrading Berkeley DB Databases +@comment node-name, next, previous, up +@section Upgrading Berkeley DB Databases + +At regular intervals, Elephant will require a specific version of Berkeley DB +because it does not parse the header files which tend to change with each release. +The patches are usually minor, but sometimes Elephant also depends on new +features of Berkeley DB. In this case, you have to update both the Berkeley DB +database files as well as any data Elephant has built within it. That creates +some special constraints for upgrading databases. + +@subsection Upgrading to 0.9 + +This section outlines how to upgrade from Elephant code base version 0.6.0 +using Berkeley DB 4.3. + +@enumerate +@item Install BDB 4.5 (keep 4.3 around for now) +@item Update my-config.sexp to point to the appropriate BDB 4.5 directories +@item Upgrade your database directory to 4.5 + @itemize + @item Run db43_recover in your 0.6 database + @item Optional: run db43_archive -d to remove all logs not part of a checkpoint + This will make catastrophic recovery impossible, but reduces the amount of data you + have to backup. + @item Backup your db files and remaining logs + @item Run db45_checkpoint -1 in your 0.6 database directory + @end itemize +@item Migrate 0.6 data to a new 0.6.1 database + @itemize + @item Open your old database: @code{(setf sc (open-store '(:BDB "/Users/me/db/ele060/")))} + @item Run upgrade: @code{(upgrade sc '(:BDB "/Users/me/db/ele061/"))} + @end itemize +@item Test your new application and report any bugs that arise to elephant-devel(a)common-lisp.net +@end itemize + +@emph{(NOTE: close-store may fail when closing the old 0.6 database, this is OK.)} + +@emph{(NOTE: 64-bit lisps will not successfully upgrade 32-bit 0.6 databases. Use a 32-bit + version of your lisp to update to 0.6.1 and then open that database in your 64-bit + lisp. There should be no compatibility problems. Best to test your application on + a 32-bit lisp if you can, just to be sure.)} + +@subsection Upgrade from Elephant 0.5 + +Follow the upgrade procedures outlined in Elephant release 0.6.0 to migrate your database +from 0.5 to 0.6. Then follow the above procedures for upgrading from an 0.6 database. + +@emph{(NOTE: It may not take much work to make 0.9 upgrade directly from 0.5 +However there are so few (none?) 0.5 users that it wasn't deemed worth +the work given that there's an upgrade path available.)} @node CL-SQL @comment node-name, next, previous, up --- /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/04/27 03:14:55 1.13 +++ /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/04/28 02:31:07 1.14 @@ -140,13 +140,15 @@ Persistent collections inherit from @ref{Class elephant:persistent-collection} and consist of the @ref{Class elephant:btree}, @ref{Class elephant:indexed-btree} and @ref{Class elephant:btree-index} classes. The following operations are defined -on most of these classes. More information can be found in @ref{Using BTrees} -and @ref{Secondary Indices}. +on most of these classes. More information can be found in @ref{Persistent BTrees} +and @ref{BTree Indexing}. @include includes/fun-elephant-make-btree.texinfo - @include includes/fun-elephant-get-value.texinfo -@include includes/fun-elephant-setf-get-value.texinfo + +Values are written to a btree using the @code{setf} method on @code{get-value}. +@c @include includes/fun-elephant-setf-get-value.texinfo + @include includes/fun-elephant-remove-kv.texinfo @include includes/fun-elephant-existsp.texinfo @include includes/fun-elephant-drop-btree.texinfo @@ -178,7 +180,8 @@ @include includes/fun-elephant-make-cursor.texinfo @include includes/fun-elephant-cursor-close.texinfo @include includes/fun-elephant-cursor-duplicate.texinfo -@include includes/fun-elephant-cursor-initialized-p.texinfo +@c fixme, not generated by docstrings +@c @include includes/fun-elephant-cursor-initialized-p.texinfo Each of the following methods return multiple values consisting of @code{(exists? key value)}. @@ -197,8 +200,8 @@ @node Index Cursor API @comment node-name, next, previous, up -@section Cursor API -@cindex Cursors +@section Index Cursor API +@cindex Cursor @cindex Index @cindex Indices @cindex API @@ -224,12 +227,15 @@ @include includes/fun-elephant-cursor-pget-both.texinfo @include includes/fun-elephant-cursor-pget-both-range.texinfo -@include includes/fun-elephant-cursor-next-dup.texinfo @include includes/fun-elephant-cursor-next-nodup.texinfo -@include includes/fun-elephant-cursor-pnext-dup.texinfo +@include includes/fun-elephant-cursor-next-dup.texinfo @include includes/fun-elephant-cursor-pnext-nodup.texinfo +@include includes/fun-elephant-cursor-pnext-dup.texinfo + @include includes/fun-elephant-cursor-prev-nodup.texinfo +@include includes/fun-elephant-cursor-prev-dup.texinfo @include includes/fun-elephant-cursor-pprev-nodup.texinfo +@include includes/fun-elephant-cursor-pprev-dup.texinfo @node Transaction API @comment node-name, next, previous, up --- /project/elephant/cvsroot/elephant/doc/scenarios.texinfo 2007/04/27 03:14:55 1.9 +++ /project/elephant/cvsroot/elephant/doc/scenarios.texinfo 2007/04/28 02:31:07 1.10 @@ -711,14 +711,15 @@ hash table indirections with a little macro: @lisp -(defmacro def-snapshot-wrapper - (accessor-name (source-classname target-classname hashname uid)) +(defmacro def-snapshot-wrapper (accessor-name + (source-classname target-classname hashname uid)) (with-gensysms (obj key ref) `(progn (defmethod ,accessorname :around ((,obj ,source-classname)) (let ((,key (call-next-method))) (when ,key (gethash ,key ,hashname)))) - (defmethod (setf ,accessorname) :around (,ref (,obj ,source-classname)) + (defmethod (setf ,accessorname) :around + (,ref (,obj ,source-classname)) (cond ((subtypep (type-of ,ref) ,target-classname) (let ((,key (find-object ,ref ,hashname))) (if ,key --- /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/04/27 03:14:55 1.19 +++ /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/04/28 02:31:11 1.20 @@ -272,7 +272,7 @@ @item @strong{Merge-conflicts in heavily multi-process/threaded situations}. This is the common read-modify-write problem in all databases. We will talk -more about this in the @ref{Transactions} section. +more about this in the @ref{Using Transactions} section. @end enumerate @@ -761,7 +761,9 @@ (get-instances-by-range 'friends 'name "Adam" "Devin") => (#<Adriana> #<Carlos>) -(get-instances-by-range 'friend 'birthday (encode-date '(1 1 1974)) (encode-date '(31 12 1984))) +(get-instances-by-range 'friend 'birthday + (encode-date '(1 1 1974)) + (encode-date '(31 12 1984))) => (#<Zaid> #<Adriana>) (mapc #'print-friend *) @@ -1207,12 +1209,12 @@ and iterating of duplicate or unique values. @item @strong{Using the Map Operators} Mapping operators can be very efficient if properly utilized. -@item @strong{Using Multiple Stores} Multiple store controllers can -be open simultaneously. However it does make the code more complex -and you need to be careful about how you use them to avoid crashes -and other unpleasant side effects. -@item @strong{Custom Transaction Architecture} You can implement your -own version of @code{with-transaction} using the +@item @strong{Using Multiple Stores} +Multiple store controllers can be open simultaneously. However it +does make the code more complex and you need to be careful about how +you use them to avoid crashes and other unpleasant side effects. +@item @strong{Custom Transaction Architecture} +You can implement your own version of @code{with-transaction} using the underlying controller methods for starting, aborting and committing transactions. You had better know what you are doing, however! @item @strong{Handling Errors and Conditions} --- /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/04/27 03:14:55 1.19 +++ /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/04/28 02:31:11 1.20 @@ -188,7 +188,7 @@ @item @strong{Merge-conflicts in heavily multi-process/threaded situations}. This is the common read-modify-write problem in all databases. We will talk -more about this in the @ref{Transactions} section. +more about this in the @ref{Transaction Details} section. @item @strong{Byte Ordering}. The primitive elements such as integers are written to disk in @@ -908,7 +908,7 @@ Once a cursor is properly initialized, it can be incremented or decremented, a simple constant-time operation on BTrees. -@code{cursor-next} and @{cursor-prev} move the cursor a single step +@code{cursor-next} and @code{cursor-prev} move the cursor a single step forward or back across the sorted key-value pairs. @code{cursor-next} moves in ascending order, @code{cursor-prev} in descending order. @@ -960,10 +960,11 @@ class indexing capabilities previously described. An index is created by using the @code{add-index} function. This -function takes the @code{indexed-btree} you wish to index, an -index-name for later retrieval, a key-form which dictates how -the index populates it's keys as a function of the main btree's -keys and values. +function takes the @code{indexed-btree} you wish to index, an symbolic +name for the index and a key-form which dictates how the index +populates it's keys as a function of the main btree's keys and values. +(It is a function of three arguments: the index itself, the key and +the value). A simple, contrived example is shown in the figure below: @@ -1018,23 +1019,37 @@ Operations that have the same behavior, but return primary btree values and keys are: -@table @samp +@multitable @columnfractions .3 .1 .35 +@headitem BTree Cursor Function @tab @tab Index Cursor Function + @item @code{cursor-first} -@code{cursor-pfirst} -@samp{@code{cursor-first}} +@tab => +@tab @code{cursor-pfirst} + @item @code{cursor-last } -@code{cursor-plast} +@tab => +@tab @code{cursor-plast} + @item @code{cursor-current } -@code{cursor-pcurrent} +@tab => +@tab @code{cursor-pcurrent} + @item @code{cursor-next } -@code{cursor-pnext} +@tab => +@tab @code{cursor-pnext} + @item @code{cursor-prev} -@code{cursor-pprev} +@tab => +@tab @code{cursor-pprev} + @item @code{cursor-set} -@code{cursor-pset} +@tab => +@tab @code{cursor-pset} + @item @code{cursor-set-range} -@code{cursor-set-prange} -@end table +@tab => +@tab @code{cursor-set-prange} +@end multitable The big difference between btree cursors and index cursors is that indices can have duplicate key values. This means we have to choose @@ -1042,100 +1057,217 @@ a duplicate segment. There are cursor operations for each: @itemize -@item Simple move. Standard btree operations plus @code{cursor-pnext} and @code{cursor-pprev}. -@item Move to next key value. @code{cursor-pnext-nodup} and @code{cursor-pprev-nodup}. -@item Move to next duplicate. +@item Simple move. Standard btree operations work plus @code{cursor-pnext} and @code{cursor-pprev} +@item Move to a different key value. @code{cursor-pnext-nodup} and @code{cursor-pprev-nodup} +@item Move to next duplicate key value. @code{cursor-pnext-dup} and @code{cursor-pprev-dup} @end itemize -@c FINISH +After incrementing through a set of duplicate items using a +@code{xxx-dup} function, the last next operation returns nil +indicating there are no more duplicates. The consequence of this is +that the cursor is now uninitialized (@code{cursor-initialized-p}) and +needs to be reset by a set or set both call. + + @node Multi-threaded Applications @comment node-name, next, previous, up @section Multi-threaded Applications -Berkeley DB plays well with threads and processes. The store -controller is thread-safe by default, that is, can be shared amongst -threads. This is enabled by the @code{:thread} keyword argument which -defaults to true. Transactions may not be shared amongst threads -except serially. One thing which is NOT thread and process safe is -recovery, which should be run when no one is else is talking to the -database environment. - -The following shared regions of Elephant are protected by standard locks: -- buffer stream pool -- access to serializer circular buffers -- writes to connection db -- where else? +Elephant is thread-safe by design. Users should not have to think +about threading except to follow a couple of simple rules. + +@enumerate +@item Do not perform transactions across multiple threads +@item Do not perform add/remove index operations on indexed-btrees +in more than one thread. +@end enumerate + +This and common coding sense should be sufficient! Elephant's +internal design for thread safety employs a number of policies +to try to minimize using lisp locks and simplify analysis of +multi threaded interactions: + +@enumerate +@item @strong{Rely on the thread safety of the data store databases} +@item @strong{Ensure transaction isolation} +@item @strong{Minimize dependency on thread-local special variables} +@item @strong{Protect shared resources for a given store controller}. +@item @strong{A use policy for shared objects (above)} +@end enumerate + +@subsection Shared Resources + +Elephant has a few shared resources which are protected by standard locks. +These are: +@itemize +@item The store controller connection table +@item The instance cache +@item The circularity buffer pool for the serializer +@item The buffer-stream pool in memutils +@end itemize + +In some cases, and on some lisp platforms, we try to use a fast lock +strategy for frequently accessed items (the resource pools and +instance cache especially). + +@subsection Data Store Thread Safety and Transactions + +Both CLSQL and Berkelely DB backends are thread safe. In CLSQL this +is by ensuring that every thread has it's own handle into the SQL +libraries or sockets. Berkeley DB is reentrant and handles locking +internally. + +Elephant depends on these guarantees especially for the isolation +properties of transactions. All operations in the context of a +given transaction should be isolated and atomic. It is important +that a transaction not be shared across threads, however. + +@subsection Minimize Dependency on Thread-Local Specials + +Elephant uses several global variables as default arguments. Most +of these were removed leaving only a couple to handle: + +@itemize +@item @strong{@code{*store-controller*}}. Store controller objects can +be shared between threads and if a user resets this variable in a local +thread to another controller, there is no problem with that either. +Users of multiple concurrent stores can specify the store controller to +all elephant API commands that don't get it from a persistent object +implicitely. +@code{*current-transaction*}. This is always set to the proper null +value globally and should not be reset in local threads. Instead, +transactions take place in a dynamic context that rebinds this variable +as a special with the current transaction. This allows for a dynamic +transaction stack for data stores that can nest transactions or when +two datastores are both doing transactions concurrently. +@end itemize -@c *** FINISH *** @node Transaction Details @comment node-name, next, previous, up @section Transaction Details +Transactions are dynamic contexts in which all side effects to +persistent slots and other persistent objects such as BTrees are +guaranteed to have the ACID properties: atomicity, consistency, +isolation and durability. On a normal exit from context, the +side effects are committed as a group. On a non-local exit, +the transaction is aborted. + +For most users, the tutorial section @ref{Using Transactions} is the +best introduction to transactions. This section adds to that by +exposing some of the details of how it is implemented. + +To reiterate, there are a few important restrictions to adhere to: + +@itemize +@item @code{*current-transaction*} is reserved for use by the transaction system. Users should not override, manipulate or close over this variable. +@item The body of a transaction cannot throw, signal or jump without aborting the transaction. Any non-local exit is considered an aborting event. Catch signals inside the transaction and return a value instead. +@item The dynamic extent of a transaction body must stay within the same thread +@end itemize + +@subsection @code{with-transaction} internals + +The @code{with-transaction} macro wraps the body expression with an +anonymous lambda expression. This closure is passed to a call to the +@code{execute-transaction} generic function which is specialized to +the current data store. + +The only bookkeeping done by the macro is ensuring that the +@code{:parent} argument is checked for the current dynamic transaction +context. If it is not owned by the default or provided store +controller, then it is not passed to @code{execute-transaction}. This +maintains a continuous dynamic stack transactions through the +with/ensure transaction macros, but allows for a single leaf +transaction to another store controller. + +Be very careful about mixing transactions between store controller. +This facility was only added to ensure that migrate worked correctly. + +The macro processes keywords arguments @code{:store-controller} +(defaults to @code{*store-controller*}), @code{:parent} (defaults to +@code{*current-transaction*}) and @code{:retries} and passes the +remaining keywords to the call to @code{execute-transaction} allowing +the user to pass data store specific transaction keywords to their +preferred data store. The consumed keywords are analyzed and then +passed on to @code{execute-transaction}. + +Any non-standard keywords for a given data store will be ignored by +other data store implementation of @code{execute-transaction} so +portable programs should not use keywords that change the semantics of +the transaction. + +@code{ensure-transaction} only calls @code{execute-transaction} if +it needs to create a fresh transaction. If the transaction in +@code{*current-transaction*} exists and belongs to the store controller +passed to @code{ensure-transaction} then it merely calls the transaction +closure, relying on the environment that created the transaction to +handle any exit procedures and determining whether to abort or commit. + +@code{*current-transaction*} contains transaction records during the +dynamic execution of a transaction. These records capture any data +store specific bookkeeping as well as the store-controller that the +transaction is associated with. + +@subsection @code{execute-transaction} internals + +See the @ref{Elephant Architecture} section for details on how +execute-transaction works. It will provide some deeper insight +into the transaction system. + +@subsection Building your own transactional framework + +Data stores are required to implement three primitive transaction +methods: @code{controller-start-transaction}, +@code{controller-abort-transaction} and +@code{controller-commit-transaction}. These are wrappers for the data +store's primitive transaction mechanism. If you use these, it is up +to you to make sure that you properly manage nested transactions, +maintain the state of @code{*current-transaction*} handle any +automated retries you might want, and handle detecting + +If you use these, you are on your own - it is easy to make mistakes with +transactions and create very complex bugs that are hard to track down. +Most users are much better off sticking with the two transaction macros +and the underlying @code{execute-transaction} method. + +@subsection Analyzing Dynamic Transaction Behavior + You can trace @code{elephant::execute-transaction} to see the sequence -of calls to @code{execute-transaction} that occur dynamically and -detect where transactions are and are not happening. We may add some -transaction diagnosis and tracing tools in the future, such as -throwing a condition when @code{with-transaction} forms are nested -dynamically. - -;; Transaction architecture: -;; -;; User and designer considerations: -;; - *current-transaction* is reserved for use by dynamic transaction context. The default global -;; value must always be null (no transaction). Each data storeend can set it to a different parameter -;; within the dynamic context of an execute-transaction. -;; - Any closures returned from within a transaction cannot bind *current-transaction* -;; - Only a normal return value will result in the transaction being committed, any non-local exit -;; results in a transaction abort. If you want to do something more sophisticated, roll your own -;; using controller-start-transaction, etc. -;; - The body of a with or ensure transaction can take any action (throw, signal, error, etc) -;; knowing that the transaction will be aborted -;; +of calls that occur dynamically and detect where and how many +transactions are and are not happening. + +@c IT WOULD BE GOOD TO EXPAND ON THIS REGARDING HOW TO SOLVE COMMON PROBLEMS -@c *** FINISH *** @node Multi-repository Operation @comment node-name, next, previous, up @section Multi-repository Operation -Elephant now keeps a small hashtables that maps ``database specifications'' into -actual database connections. +Elephant maintains a small hashtable that maps ``database +specifications'' into actual @code{store-controller} objects. + +The basic strategy is that the ``database specification'' object is +stored in every persistent object and collection so that the +repository can be found. In this way, objects that reside in +different repositories can coexist within the LISP object space, +allowing data migration or multiple user stores. + +All persistent instances store their oid and a store-controller +reference in internal slots. Slot access and other protocols use +this to provide access. This executes an auto-transaction or joins +a surrounding transaction if the @code{transaction-record} in +@code{*current-transaction*} matches the store. + +When operating with multiple stores and nested transactions there are +some subtle issues to work around: how to avoid writing one store with +a transaction created in the context of another. A nested or ensured +transaction is only indicated in the call to +@code{execute-transaction} if the store controllers match, otherwise a +new transaction for that store is created. + +@c A PICTURE OF THE DYNAMIC CONTEXT WOULD BE USEFUL HERE -If a database spec is a string, it is assumed to be a BerkeleyDB path. -If it is a list, it is a assumed to be a CLSQL connection specification. -For example: -@lisp -ELE-TESTS> *testdb-path* -"/home/read/projects/elephant/elephant/tests/testdb/" -ELE-TESTS> *testpg-path* -(:postgresql "localhost.localdomain" "test" "postgres" "") -ELE-TESTS> -@end lisp -The tests now have a function @code{do-all-tests-spec} that take a spec and -based on its type attempt to open the correct kind of store controller and -perform the tests. - -The routine @code{get-controller} takes this specifiation. - -The basic strategy is that the ``database specification'' object is stored in -every persistent object and collection so that the repository can be found. - -In this way, objects that reside in different repositories can coexist within -the LISP object space, allowing data migration. - -;; Multiple stores - -;; Multiple store considerations: -;; - When operating with multiple stores, nested transactions and BDB there are some subtle issues to -;; work around: how to avoid writing one store with a transaction created in the context of another. -;; - For many leaf functions: *store-controller* and *current-transaction* have to both be correct; -;; this requirement may relax in the future -;; - The following macros accomodate multiple stores by requiring that execute-transaction return a -;; pair of (store-controller . txn-obj) where txn-obj is owned by the backend and the store-controller -;; is the store instance it is associated with. A nested or ensured transaction is only indicated -;; in the call to execute transaction if the store controllers match, otherwise a new transaction -;; for that store is created @node Multiple Processes and Distributed Applications @comment node-name, next, previous, up

1 0

[elephant-cvs] CVS elephant
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv16753 Modified Files: CREDITS README Removed Files: BEN-RETIRING-TODO.txt ChangeLog INSTALL NEWS NOTES TODO TUTORIAL UPGRADE-BDB Log Message: Cleaning up root directory files; map-index performance enhancement, index api cleanup, ensure transaction fix, alpha quality documentation draft --- /project/elephant/cvsroot/elephant/CREDITS 2006/11/11 06:27:37 1.8 +++ /project/elephant/cvsroot/elephant/CREDITS 2007/04/28 02:31:06 1.9 @@ -57,3 +57,16 @@ Marco Baringer provided a partial patch for 64-bit support. +Pierre Thierry for a *features* hack to turn off optimization as well as +many suggestions and dicussions. + +Henrik Hjelte for a number of bug fixes and for testing as well as +his wonderful postmodern data store! + +Edi Weisz for Lispwork and Win32 patches + +Mac Chan for his Hutchentoot blog tutorial + +And many others for their comments, testing and suggestions -- we're sorry +if we missed crediting you here. + --- /project/elephant/cvsroot/elephant/README 2006/11/11 06:27:37 1.6 +++ /project/elephant/cvsroot/elephant/README 2007/04/28 02:31:06 1.7 @@ -1,89 +1,58 @@ -============================================== -Elephant -- an object database for Common Lisp -============================================== +================================================================================= +Elephant -- An persistent metaobject protocol and object database for Common Lisp +================================================================================= ------------- -Introduction ------------- - -Elephant is an object database for Common Lisp. It supports -storing CLOS objects and most lisp primitives, and access to -BTrees. It uses Sleepycat / Berkeley DB, a -widely-distributed embedded database; many unix systems have -it installed by default. Sleepycat is server-less, ACID -compliant, transactional, process and thread safe, and fast -relative to relational databases; hopefully Elephant -inherits these properties. - -Goals: - -* Transparency: most Lisp values are easy to persist without -much effort or special syntax. Talk to the DB with Lisp -code, not SQL or another domain-specific language. No -additional server to run. - -* Safety: ACID, transactions. Concurrent with good -multi-user and -thread semantics / isolation, locking and -deadlock detection. - -* Simplicity: be a small library with few surprises for the -programmer. Lisp and Berkeley DB together are an excellent -substrate, try to use their features as much as possible. - -* Performance: leverage Sleepycat performance and -reliability. In addition to fast concurrent / transactional -modes, elephant will (eventually) offer an accellerated -single-user mode. - -Join the Elephant mailing lists to ask your questions and -receive updates. They're on the Elephant website +Detailed information on Elephant with an introduction, supported platforms, +latest news and documentation can be found at: http://www.common-lisp.net/project/elephant ------------------- -Supported Lisps ------------------- -CMUCL Linux (lightly tested) -SBCL Linux Mac <future?> (heavily tested) -Allegro Linux Mac cygwin? (heavily tested) -CLISP Linux Mac cygwin (lightly tested) -MCL Mac (lightly tested) - ----------------------------- Copyright, License + Warrenty ----------------------------- -See LICENSE. +See LICENSE + +------------------- +Supported Platforms +------------------- + +See the website: http://www.common-lisp.net/project/elephant/platforms.html ------------ -Installation +Latest News ------------ -See INSTALL +See the website: http://www.common-lisp.net/project/elephant/news.html ----------------------- -Tutorial and Reference ----------------------- +-------------- +Documentation +-------------- -HTML docs and texinfo sources can be found in the docs/ directory. +HTML and PDF documentation can be found on the website, or built via a +Makefile (and SBCL) in the /doc directory + +------------ +Installation +------------ + +See the documentation, Chapter 3 (Installation) ------------------------ Design and Development ------------------------ -See NOTES for some design internals. +See the documentation, Chapter 7 (Elephant Architecture) and +Chapter 8 (Data Store API) for some design internals. + +If you want to contribute, see the Trac website for current +feature release plans and other things that need to be done. -If you want to contribute, see TODO for current feature release plans -and other things that need to be done. +http://trac.common-lisp.net/elephant ------- Authors ------- -See CREDITS. - ----- -News ----- +See CREDITS and LICENSE -See NEWS and ChangeLog. \ No newline at end of file

1 0

[elephant-cvs] CVS elephant/doc/oldfiles
by ieslick 28 Apr '07

28 Apr '07

Update of /project/elephant/cvsroot/elephant/doc/oldfiles In directory clnet:/tmp/cvs-serv16495/oldfiles Log Message: Directory /project/elephant/cvsroot/elephant/doc/oldfiles added to the repository

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 27 Apr '07

27 Apr '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv32343/src/elephant Modified Files: classes.lisp collections.lisp package.lisp query.lisp transactions.lisp Log Message: Bug fixes to change-class; drop-btree; enable :from-end and :collect on map-btree (not map-class though); export and documentation edits --- /project/elephant/cvsroot/elephant/src/elephant/classes.lisp 2007/04/25 02:28:01 1.32 +++ /project/elephant/cvsroot/elephant/src/elephant/classes.lisp 2007/04/27 13:32:16 1.33 @@ -185,13 +185,17 @@ ;; CLASS CHANGE PROTOCOL ;; -(defmethod change-class ((inst persistent) (class t) &rest rest) - (cerror "Ignore and continue?" - "Changing a persistent instance's class to a non-persistent class is not currently allowed")) - -(defmethod change-class ((inst standard-object) (class persistent-metaclass) &rest rest) - (cerror "Ignore and continue?" - "Changing a standard instance to a persistent instance is not supported")) +(defmethod change-class :around ((previous persistent) (new-class standard-class) &rest initargs) + (declare (ignorable initargs)) + (unless (subtypep (type-of new-class) 'persistent-metaclass) + (error "Persistent instances cannot be changed to non-persistent classes in change-class")) + (call-next-method)) + +(defmethod change-class :around ((previous standard-object) (new-class persistent-metaclass) &rest initargs) + (declare (ignorable initargs)) + (unless (subtypep (type-of previous) 'persistent) + (error "Standard classes cannot be changed to non-persistent classes in change-class")) + (call-next-method)) (defmethod update-instance-for-different-class :around ((previous persistent) (current persistent) &rest initargs &key) (let* ((old-class (class-of previous)) --- /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/25 02:28:01 1.26 +++ /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/04/27 13:32:17 1.27 @@ -70,6 +70,18 @@ (defmethod optimize-layout ((bt t) &key &allow-other-keys) t) +(defgeneric drop-btree (bt) + (:documentation "Delete all key-value pairs from the btree and + render it an invalid object in the data store")) + +(defmethod drop-btree ((bt btree)) + (ensure-transaction (:store-controller *store-controller*) + (with-btree-cursor (cur bt) + (loop for (exists? key) = (multiple-value-list (cursor-first cur)) + then (multiple-value-list (cursor-next cur)) + while exists? + do (remove-kv key bt))))) + ;; ;; Btrees that support secondary indices ;; @@ -161,17 +173,17 @@ (defclass cursor () ((oid :accessor cursor-oid :type fixnum :initarg :oid) -;; (intialized-p cursor) means that the cursor has -;; a legitimate position, not that any initialization -;; action has been taken. The implementors of this abstract class -;; should make sure that happens under the sheets... -;; According to my understanding, cursors are initialized -;; when you invoke an operation that sets them to something -;; (such as cursor-first), and are uninitialized if you -;; move them in such a way that they no longer have a legimtimate -;; value. (initialized-p :accessor cursor-initialized-p - :type boolean :initform nil :initarg :initialized-p) + :type boolean :initform nil :initarg :initialized-p + :documentation "Predicate indicating whether +the btree in question is initialized or not. Initialized means +that the cursor has a legitimate position, not that any +initialization action has been taken. The implementors of this +abstract class should make sure that happens under the +sheets... Cursors are initialized when you invoke an operation +that sets them to something (such as cursor-first), and are +uninitialized if you move them in such a way that they no longer +have a legimtimate value.") (btree :accessor cursor-btree :initarg :btree)) (:documentation "A cursor for traversing (primary) BTrees.")) @@ -240,13 +252,13 @@ (defgeneric cursor-delete (cursor) (:documentation - "Delete by cursor. The cursor is at an invalid position -after a successful delete.")) + "Delete by cursor. The cursor is at an invalid position, +and uninitialized, after a successful delete.")) (defgeneric cursor-put (cursor value &key key) (:documentation - "Put by cursor. Currently doesn't properly move the -cursor.")) + "Overwrite value at current cursor location. Currently does + not properly move the cursor.")) (defclass secondary-cursor (cursor) () (:documentation "Cursor for traversing secondary indices.")) @@ -354,7 +366,7 @@ (defun lisp-compare-equal (a b) (equal a b)) -(defgeneric map-btree (fn btree &rest args &key start end value &allow-other-keys) +(defgeneric map-btree (fn btree &rest args &key start end value from-end collect &allow-other-keys) (:documentation "Map btree maps over a btree from the value start to the value of end. If values are not provided, then it maps over all values. BTrees do not have duplicates, but map-btree can also be used with indices @@ -365,26 +377,47 @@ ;; function orders by type tag and nil is the highest valued type tag so nils are the last ;; possible element in a btree ordered by value. -(defmethod map-btree (fn (btree btree) &rest args &key start end (value nil value-set-p) &allow-other-keys) - (let ((end (if value-set-p value end))) +(defmethod map-btree (fn (btree btree) &rest args &key start end (value nil value-set-p) + from-end collect &allow-other-keys) + (let ((end (if value-set-p value end)) + (results nil)) (ensure-transaction (:store-controller (get-con btree) :degree-2 *map-using-degree2*) (with-btree-cursor (curs btree) - (multiple-value-bind (exists? key value) - (cond (value-set-p - (cursor-set curs value)) - ((null start) - (cursor-first curs)) - (t (cursor-set-range curs start))) - (if exists? - (funcall fn key value) - (return-from map-btree nil)) - (loop - (multiple-value-bind (exists? k v) - (cursor-next curs) - (declare (dynamic-extent exists? k v)) - (if (and exists? (or (null end) (lisp-compare<= k end))) - (funcall fn k v) - (return nil))))))))) + (flet ((continue-p (key) + ;; Do we go to the next value? + (or (if from-end (null start) (null end)) + (if from-end + (or (not (lisp-compare<= key start)) + (lisp-compare-equal key start)) + (lisp-compare<= key end)))) + (collector (k v) + (push (funcall fn k v) results))) + (let ((fn (if collect #'collector fn))) + (declare (dynamic-extent (function continue-p) (function collector))) + (multiple-value-bind (exists? key value) + (cond (value-set-p + (cursor-set curs value)) + ((and (not from-end) (null start)) + (cursor-first curs)) + ((and from-end (null end)) + (cursor-last curs)) + (t (if from-end + (cursor-set-range curs end) + (cursor-set-range curs start)))) + (declare (dynamic-extent exists? k v)) + (if exists? + (funcall fn key value) + (return-from map-btree nil)) + (loop + (multiple-value-bind (exists? k v) + (if from-end + (cursor-prev curs) + (cursor-next curs)) + (declare (dynamic-extent exists? k v)) + (if (and exists? (continue-p k)) + (funcall fn k v) + (return nil))))))))) + results)) (defgeneric map-index (fn index &rest args &key start end value from-end collect &allow-other-keys) (:documentation "Map-index is like map-btree but for secondary indices, it --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/25 02:28:02 1.33 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/04/27 13:32:17 1.34 @@ -216,12 +216,11 @@ #:btree-index #:add-index #:get-index #:remove-index #:map-indices #:get-primary-key #:primary #:key-form #:key-fn - #:with-btree-cursor #:map-btree #:map-index + #:with-btree-cursor #:map-btree #:map-index #:drop-btree #:empty-btree-p #:dump-btree #:btree-keys #:btree-differ-p #:cursor #:secondary-cursor #:make-cursor #:make-simple-cursor - #:cursor-close #:cursor-init - #:cursor-duplicate #:cursor-current #:cursor-first + #:cursor-close #:cursor-duplicate #:cursor-current #:cursor-first #:cursor-last #:cursor-next #:cursor-next-dup #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup #:cursor-set #:cursor-set-range #:cursor-get-both @@ -229,8 +228,8 @@ #:cursor-pcurrent #:cursor-pfirst #:cursor-plast #:cursor-pnext #:cursor-pnext-dup #:cursor-pnext-nodup #:cursor-pprev #:cursor-pprev-nodup #:cursor-pset - #:cursor-pset-range #:cursor-pget-both - #:cursor-pget-both-range + #:cursor-pset-range #:cursor-pget-both #:cursor-pget-both-range + #:cursor-initialized-p #:find-class-index #:find-inverted-index #:enable-class-indexing #:disable-class-indexing @@ -240,7 +239,7 @@ #:report-indexed-classes #:class-indexedp-by-name - #:map-class #:map-class-index + #:map-class #:map-inverted-index #:get-instances-by-class #:get-instance-by-value #:get-instances-by-value --- /project/elephant/cvsroot/elephant/src/elephant/query.lisp 2007/04/12 02:47:33 1.4 +++ /project/elephant/cvsroot/elephant/src/elephant/query.lisp 2007/04/27 13:32:17 1.5 @@ -73,8 +73,8 @@ (if (find-inverted-index class slot) (if (= (length values) 1) (progn - (map-class-index fn class slot (first values) (first values)) - (map-class-index fn class slot (first values) (second values)))) + (map-inverted-index fn class slot (first values) (first values)) + (map-inverted-index fn class slot (first values) (second values)))) (map-class #'filter-by-relation class)) (map-class-query #'filter-by-relation (cdr constraints)))))) --- /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/12 02:47:33 1.11 +++ /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/04/27 13:32:17 1.12 @@ -95,13 +95,18 @@ ;; form)) (defun transaction-object-p (txnrec) - (consp txnrec)) + (and (not (null txnrec)) + (consp txnrec) + (subtypep (type-of (car txnrec)) 'store-controller))) (defun owned-txn-p (sc parent-txn-rec) (and parent-txn-rec (transaction-object-p parent-txn-rec) (eq sc (transaction-store parent-txn-rec)))) +(define-condition transaction-retry-count-exceeded () + ((retry-count :initarg :count))) + (defmacro with-transaction ((&rest keyargs &key (store-controller '*store-controller*) (parent '*current-transaction*) @@ -126,7 +131,7 @@ (defmacro ensure-transaction ((&rest keyargs &key (store-controller '*store-controller*) - (transaction '*current-transaction*) + (parent '*current-transaction*) (retries 200) &allow-other-keys) &body body) @@ -139,7 +144,7 @@ (sc (gensym))) `(let ((,txn-fn (lambda () ,@body)) (,sc ,store-controller)) - (if (owned-txn-p ,sc ,transaction) + (if (owned-txn-p ,sc ,parent) (funcall ,txn-fn) (funcall #'execute-transaction ,store-controller ,txn-fn

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 27 Apr '07

27 Apr '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv32343/src/db-bdb Modified Files: bdb-transactions.lisp Log Message: Bug fixes to change-class; drop-btree; enable :from-end and :collect on map-btree (not map-class though); export and documentation edits --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/25 02:27:58 1.14 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/04/27 13:32:14 1.15 @@ -57,7 +57,12 @@ (db-transaction-abort txn))))) (when success (return (values-list result))))) - finally (error "Too many retries in transaction")))) + finally (cerror "Retry transaction again?" + 'transaction-retry-count-exceeded + :format-control "Transaction exceeded the ~A retries limit" + :format-arguments (list retries) + :count retries)))) + (defmethod controller-start-transaction ((sc bdb-store-controller) &key

1 0

[elephant-cvs] CVS elephant
by ieslick 27 Apr '07

27 Apr '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv32343 Modified Files: TODO Log Message: Bug fixes to change-class; drop-btree; enable :from-end and :collect on map-btree (not map-class though); export and documentation edits --- /project/elephant/cvsroot/elephant/TODO 2007/04/25 02:27:56 1.81 +++ /project/elephant/cvsroot/elephant/TODO 2007/04/27 13:32:11 1.82 @@ -31,10 +31,6 @@ - Add notes about deadlock-detect - Add notes about checkpoint (null in SQL?) -Additional features to document in Trac: -- Add error/restart conditions when lose data on: class redef, index drop, etc. -- :association slot option? - 0.6.1 - Features COMPLETED to date ----------------------------------

1 0

[elephant-cvs] CVS elephant/doc
by ieslick 27 Apr '07

27 Apr '07

Update of /project/elephant/cvsroot/elephant/doc In directory clnet:/tmp/cvs-serv2708 Modified Files: data-store-reference.texinfo docstrings.lisp elephant-design.texinfo elephant.texinfo installation.texinfo reference.texinfo scenarios.texinfo tutorial.texinfo user-guide.texinfo Log Message: Latest edits, plus integration with Roberts --- /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/04/02 00:51:06 1.6 +++ /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/04/27 03:14:55 1.7 @@ -8,16 +8,16 @@ This reference includes functions that need to be overridden, classes inherited from or other action taken to implement support for a new -data store backend. Included are the exported elephant functions that -need methods defined on them as well as the backend-only functions -exported in backends.lisp. Some functions here are utilities from the +data store. Included are the exported elephant functions that +need methods defined on them as well as the data-store-only functions +exported in data-store-api.lisp. Some functions here are utilities from the main elephant package that support store implementations, but are not required. Migration, class indices and query interfaces are implemented on top of the store API and require no special support by implementors. -Because the number of backend implementors is small, this is a minimal -documentation set intended to serve as an initial guide and a +Because the number of data store implementors is small, this is a +minimal documentation set intended to serve as an initial guide and a reference. However, it is anticipated that some interaction will be needed with the developers to properly harden a datastore for release. @@ -25,7 +25,7 @@ relevant to them. @menu -* Registration: DSR Registration. Register the backend to parse controller specifications. +* Registration: DSR Registration. Register the data store for the appropriate controller specifications. * Store Controllers: DSR Store Controllers. Subclassing the store controller. * Handling Serialization: DSR Handling Serialization. Available facilities for serializing objects. * Memory Utilities: DSR Memory Utilities. Writing primitive C types. @@ -43,22 +43,22 @@ @cindex Registration and Initialization Elephant looks at the first element of the specification list to -determine which backend code base to use. The master table for this -information is @code{*elephant-backends*} in elephant/controller.lisp. -This will need to be augmented for every backend with the +determine which data store module to use. The master table for this +information is @code{*elephant-data-stores*} in elephant/controller.lisp. +This will need to be augmented for every data store with the specification keyword tag to be used (such as @code{:BDB} or @code{:CLSQL}) and the required asdf dependencies. -In addition, the backend source should use an eval-when statement to +In addition, the data store source should use an eval-when statement to call the following function: -@include includes/fun-elephant-backend-register-backend-con-init.texinfo +@include includes/fun-elephant-data-store-register-data-store-con-init.texinfo -If the backend requires any special user-specified configuration, +If the data store requires any special user-specified configuration, augment the key types in config.sexp with what you need and use the following function to access. -@include includes/fun-elephant-backend-get-user-configuration-parameter.texinfo +@include includes/fun-elephant-data-store-get-user-configuration-parameter.texinfo @node DSR Store Controllers @comment node-name, next, previous, up @@ -69,25 +69,25 @@ which are called by open-store and close-store respectively. @include includes/class-elephant-store-controller.texinfo -@include includes/fun-elephant-backend-open-controller.texinfo -@include includes/fun-elephant-backend-close-controller.texinfo -@include includes/fun-elephant-backend-connection-is-indeed-open.texinfo +@include includes/fun-elephant-data-store-open-controller.texinfo +@include includes/fun-elephant-data-store-close-controller.texinfo +@include includes/fun-elephant-data-store-connection-is-indeed-open.texinfo For upgrading and opening legacy databases it is important that a store be able to indicate which version of elephant was used to create it. This governs the chosen serializer, mappings between elephant symbols used in an old vs. new version, etc. Because this is called to initialize the serializer, it must directly implemented by the -backend without using the serializer. +data store without using the serializer. -@include includes/fun-elephant-backend-database-version.texinfo +@include includes/fun-elephant-data-store-database-version.texinfo There are some utilities for serializing simple data without a serializer using the memutil package. -@include includes/fun-elephant-backend-serialize-database-version-key.texinfo -@include includes/fun-elephant-backend-serialize-database-version-value.texinfo -@include includes/fun-elephant-backend-deserialize-database-version-value.texinfo +@include includes/fun-elephant-data-store-serialize-database-version-key.texinfo +@include includes/fun-elephant-data-store-serialize-database-version-value.texinfo +@include includes/fun-elephant-data-store-deserialize-database-version-value.texinfo @node DSR Persistent Objects and Slot Access @@ -106,22 +106,22 @@ that functions such as map-btree do not need a store-controller argument. (NOTE: Should this function be user visible?) -@include includes/fun-elephant-backend-get-con.texinfo +@include includes/fun-elephant-data-store-get-con.texinfo All objects require a unique object identifier. During new object -creation the backend is asked to produce a unique id. +creation the data store is asked to produce a unique id. -@include includes/fun-elephant-backend-next-oid.texinfo +@include includes/fun-elephant-data-store-next-oid.texinfo These functions are called by the metaclass protocol to implement the appropriate operations on persistent class slots. Unless protected by a transaction, the side effects of these functions should be atomic, persistent and visible to other threads on completion. -@include includes/fun-elephant-backend-persistent-slot-writer.texinfo -@include includes/fun-elephant-backend-persistent-slot-reader.texinfo -@include includes/fun-elephant-backend-persistent-slot-boundp.texinfo -@include includes/fun-elephant-backend-persistent-slot-makunbound.texinfo +@include includes/fun-elephant-data-store-persistent-slot-writer.texinfo +@include includes/fun-elephant-data-store-persistent-slot-reader.texinfo +@include includes/fun-elephant-data-store-persistent-slot-boundp.texinfo +@include includes/fun-elephant-data-store-persistent-slot-makunbound.texinfo @node DSR Collections @comment node-name, next, previous, up @@ -137,9 +137,9 @@ @include includes/class-elephant-btree-index.texinfo @include includes/class-elephant-indexed-btree.texinfo -To create the backend-appropriate type of btree, the backend +To create the data store-appropriate type of btree, the data store implements this method (and possibly related methods) aginst their store-controller. -@include includes/fun-elephant-backend-build-btree.texinfo +@include includes/fun-elephant-data-store-build-btree.texinfo Most of the user-visible operations over BTrees must be implemented. Class indexing functions such as @code{map-class} and @@ -181,19 +181,19 @@ These functions must be implemented or stubbed by all data stores. -@include includes/fun-elephant-backend-execute-transaction.texinfo +@include includes/fun-elephant-data-store-execute-transaction.texinfo -@include includes/fun-elephant-backend-controller-start-transaction.texinfo -@include includes/fun-elephant-backend-controller-commit-transaction.texinfo -@include includes/fun-elephant-backend-controller-abort-transaction.texinfo +@include includes/fun-elephant-data-store-controller-start-transaction.texinfo +@include includes/fun-elephant-data-store-controller-commit-transaction.texinfo +@include includes/fun-elephant-data-store-controller-abort-transaction.texinfo These are supporting functions and variables for implementing transactions. -@include includes/var-elephant-backend-star-current-transaction-star.texinfo -@include includes/fun-elephant-backend-make-transaction-record.texinfo -@include includes/fun-elephant-backend-transaction-store.texinfo -@include includes/fun-elephant-backend-transaction-object.texinfo +@include includes/var-elephant-data-store-star-current-transaction-star.texinfo +@include includes/fun-elephant-data-store-make-transaction-record.texinfo +@include includes/fun-elephant-data-store-transaction-store.texinfo +@include includes/fun-elephant-data-store-transaction-object.texinfo ;; Designer considerations: ;; - with-transaction passes *current-transaction* or the user parameter to execute-transaction @@ -229,7 +229,7 @@ @section Handling Serialization @cindex Serialization -Backends must initialize @ref{Class elephant:store-controller} with +Data stores must initialize @ref{Class elephant:store-controller} with internal serializer functions. Packages @code{elephant-serializer1} and @code{elephant-serializer2} contains serialize and deserialize methods on buffer-streams as defined in @code{elephant-memutil}. The @@ -240,18 +240,18 @@ NOTE: This should perhaps become entirely the job of the data store to decide how to serialize values and for a specific version, what serializer to use. The elphant main package can define serializers -for use by different backends. +for use by different data stores. @end verbatim -@include includes/fun-elephant-backend-serialize.texinfo -@include includes/fun-elephant-backend-deserialize.texinfo +@include includes/fun-elephant-data-store-serialize.texinfo +@include includes/fun-elephant-data-store-deserialize.texinfo These utility functions are useful if a data store does not have the ability to store variable length binary data. They are based on the @code{cl-base64} library. -@include includes/fun-elephant-backend-serialize-to-base64-string.texinfo -@include includes/fun-elephant-backend-deserialize-from-base64-string.texinfo +@include includes/fun-elephant-data-store-serialize-to-base64-string.texinfo +@include includes/fun-elephant-data-store-deserialize-from-base64-string.texinfo @node DSR Memory Utilities @comment node-name, next, previous, up --- /project/elephant/cvsroot/elephant/doc/docstrings.lisp 2004/09/19 17:44:43 1.1 +++ /project/elephant/cvsroot/elephant/doc/docstrings.lisp 2007/04/27 03:14:55 1.2 @@ -759,7 +759,9 @@ escaped in symbol names, but if a docstring contains invalid Texinfo markup, you lose." (handler-bind ((warning #'muffle-warning)) - (let ((directory (merge-pathnames (pathname directory)))) + (let ((directory (merge-pathnames (pathname directory))) + (*print-right-margin* 200)) + (declare (special *print-miser-width*)) (ensure-directories-exist directory) (dolist (package packages) (dolist (doc (collect-documentation (find-package package))) --- /project/elephant/cvsroot/elephant/doc/elephant-design.texinfo 2007/04/21 17:22:35 1.4 +++ /project/elephant/cvsroot/elephant/doc/elephant-design.texinfo 2007/04/27 03:14:55 1.5 @@ -1,5 +1,6 @@ @c -*-texinfo-*- +@c *** FINISH *** @node Elephant Architecture @comment node-name, next, previous, up @chapter Elephant Architecture @@ -10,17 +11,13 @@ support easy upgrading, repository migration, shared functionality between data stores and general hygene. -The current architecture is modularized into the following pieces: +The architecture has been carefully modularized: -@verbatim -[Picture describing] Metaclass, Store controller, persistent objects, -data stores, serializer & memutils. Derived features such as class -indexing, migration, query system and root operations can also be -illustrated? -@end verbatim +(a)image{ArchDiagram,,4.5in,[Architecture Diagram],.png} To get a feeling for what is happening inside elephant, it is probably -most useful to walk through the various major protocols: +best to walk through the various major protocols to see how these +components participate in implementing them. @itemize @item Initialization of a store controller @@ -32,23 +29,247 @@ @section Initializing a store controller -When the main elephant @code{open-store} function is called, it calls -@code{get-controller} which grabs an existing store controller if the -spec is identical, or builds a new controller. Building the -controller requires loading any dependencies via asdf, calling a -backend initialization function (if it is the first instance of that -backend being created), and then calling an initialization function -that returns a @code{store-controller} subclass instance specific to -that backend. +When the main elephant @code{open-store} function is called with a +specification, it calls get-controller which first checks to see if a +controller already exists for that spec. + +If there is no controller, it calls @code{build-controller} to +construct one. If the data store code base is not present, +@code{load-data-store} is called to ensure that any asdf dependencies +are satisfied. The associations for asdf dependencies are statically +configured in @code{*elephant-data-stores*} for each data store type +supported by elephant. + +While being loaded, the data store is responsible for calling +@code{register-data-store-con-init} to register a data store +initialization function for its spec type (i.e. :BDB or :CLSQL). +For example, from bdb-controller.lisp: + +@lisp +(eval-when (:compile-toplevel :load-toplevel) + (register-data-store-con-init :bdb 'bdb-test-and-construct)) +@end lisp + +This mapping between spec types and initialization functions is +accessed by @code{lookup-data-store-con-init} from within +@code{build-controller}. The function returned by +@code{lookup-data-store-con-init} is passed the full specification and +returns a @code{store-controller} subclass instance for the specified +data store. + +The new controller is stored in the @code{*dbconnection-spec*} hash +table, associating the object with its specification. Finally +Elephant calls open-controller to actually establish a connection to +or create the files of the data store. + +Finally, if the default store controller @code{*store-controller*} is +nil, it will be initialized with the new store controller, otherwise +the original value is left in @code{*store-controller*} until that +store controller is closed using @code{close-store}. -Elephant than calls open-controller to actually establish a connection -to or create the files of the data store. +The data store implementor has access to various utilities to aid +initialization. + +@itemize +@item @code{get-user-configuration-parameter} - Access symbol tags + in my-config.sexp to access data store specific user +configuration. You can also add special variables to variables.lisp +and add a tag-variable pair to @code{*user-configurable-parameters*} +in variables.lisp to automatically initialize it when the store +controller is opened. +@item @code{get-con} behavior when store is closed or lost +@item @code{database-version} a store controller implements this + in order to tell Elephant what serializer to use. Currently, + version 0.6.0 databases use serializer1 and all later database + use serializer version 2. This is to ensure that a given version + of the Elephant code can open databases from prior versions in + order to properly upgrade to the new code base. +@item Symbol conversions. To aid in opening legacy databases, a + symbol conversion facility is provided in controller.lisp to + be applied to any symbols extracted from the legacy data store. + (if, for instance, the type name of subclasses changed, such as + sleepycat-btree becoming bdb-btree) +@end itemize + +At this point, all operations referencing the store controller should +be able to proceed. + +At the end of a session, @section Persistent Object Creation + +The only thing that a data store has to do to support new object +creation, other than implement the slot protocol, is implement the +method @code{next-oid} to return the next unique object id for the +persistent object being created. + +Existing objects are created during deserialization of object +references. The serializer subsystem is built-into the core of +elephant and can be used by data stores. The serializer is abstracted +so that multiple serializers can be co-resident and the data store can +choose the appropriate one. The abstraction boundary between the +serializer, the data store, and the core Elephant system is not +perfect, so be aware and refer to existing data store implementations +if in doubt. + +A serializer takes as arguments the store-controller, lisp object and +a @code{buffer-stream} from the memory utility library and returns the +buffer-stream with the binary serialized object. The deserializer +reverses this process. For all lisp objects except persistent +classes, this means reallocating the storage space for the object and +recreating all its contents. Deserializing a standard object results +in a new standard object of the same class with the same slot values. + +Persistent classes are dealt with specially. When a persistent object +is serialized, it's oid and class are stored in the +@code{buffer-stream}. On deserialization it uses the oid to check in +the store-controller's cache for an existing placeholder object. If +the cache misses, then it creates a new placeholder object using the +class and oid as described in @xref{Persistent Classes and Objects}. +The store controller contains a cache instance that is automatically +initialized by the core Elephant object protocol. + +Currently the serializer is selected by the core Elephant code based +on the store controller's database version. See the reference section +for details on implementing the store-controller database version +method. It is a relatively small change to have the data store choose +its own serializer, however we will have to tighten up and document +the contracts between the Elephant core code, serializer and data store. + @section Persistent Slot Protocol -@section Persistent Slot Protocol -@section Persistent Collection Protocol + +The core protocol that the data store needs to support is the slot +access protocol. During object initialization, these functions are +called to initialize the slots of the object. The four functions are: + +@itemize +@item @code{persistent-slot-reader} +@item @code{persistent-slot-writer} +@item @code{persistent-slot-boundp} +@item @code{persistent-slot-makunbound} +@end itemize + +More details can be found in the data store api reference section. In +short, these functions specialize on the specific @code{store-controller} of +the data store and take instances, values and slotnames as appropriate. + +Typically the oid will be extracted from the instance and be used to +update a table or record where the oid and slotname identifies the +value. A slot is typically unbound when no value exists (as opposed to +nil). + +@section Persistent Collection Protocols + +The BTree protocol is the most extensive interface that data stores must +implement. Data store implementations are required to subclass the +abstract classes @code{btree}, @code{indexed-btree}, and @code{index} +and implement their complete APIs. Each class type is constructed +by Elephant using a @code{store-controller} that builds them. These +methods are @code{build-btree}, @code{build-indexed-btree} and +@code{build-index}. + +The @code{get-value} interface is similar to the persistent +slot reader and writer, but instead of using oid and slotname to set +values, it uses the btree oid and a key value as a unique identifier +for a value. + +The BTree protocol almost requires an actual BTree implementation to +be at all efficient. Keys and values need to be accessible via the +cursor API, which means they need to be walked linearly in the sort +order of the keys (described in @xref{Persistent BTrees}). + +An indexed BTree automatically maintains a hash table of the indices +defined on it so that users can access them by mapping or +lookup-by-name. The data store also has access to this interface. + +A BTree index must also maintain a connection to its parent BTree so +that an index value can be used as a primary tree key to retrieve the +primary BTree value as part of the @code{cursor-pnext} and +@code{cursor-pprev} family of methods. + +The contract of @code{remove-kv} is that the storage in the data store +is actually freed for reuse. + +Persistent set implemenation is optional. A default BTree based +implementation is provided by default + @section Implementing Transactions +One of the most important pieces of functionality remaining to discuss +is implementing transactions. In existing data stores, transactions +are merely extensions of the underlying start, commit and abort +methods of the 3rd party library or server being used. The Elephant +user interfaces to these functions in two ways: a call to +@code{execute-transaction} or explicit calls to @code{controller-start-transaction}, +@code{controller-commit-transaction} and @code{controller-abort-transaction}. + +@subsection Implementing Execute Transaction + +The macros @code{with-transaction} and @code{ensure-transaction} wrap +access to the data store's @code{execute-transaction}. This function +has a rich contract. It accepts as arguments the store controller, a +closure that executes the transaction body and a set of keywords. +Keywords required to be supported by the method (or ignored without +loss of semantics) are @code{:parent} and @code{:retries}. + +The semantics of @code{with-transaction} are that a new transaction +will always be requested of the data store. If a transaction exists, +@code{ensure-transaction} will merely call the transaction closure. +If not it will function as a call to @code{with-transaction}. + +@code{execute-transaction} is that it must ensure that the transaction +closure is executed within a dynamic context that insures the ACID +properties of any database operations (@code{pset},@code{btree} or +persistent slot operations). If there is a non-local exit during this +execution, the transaction should be aborted. If it returns normally, +the transaction is committed. The integer in the @code{:retries} +argument dictates how many times @code{execute-transaction} should +retry the transaction before failing. + +Elephant provides some bookkeeping to the data store to help with +nested transactions by using the @code{*current-transaction*} dynamic +variable. In the dynamic context of the transaction closure, another +call to @code{execute-transaction} may occur with the transaction +argument defaulting to the value of @code{*current-transaction*}. The +data store has to decide how to handle these cases. To support this, +the first call to execute transaction can create a dynamic binding for +@code{*current-transaction*} using the @code{make-transaction-record} +call. This creates a transaction object that records the store +controller that started the transaction and any data store-specific +transaction data. + +The current policy is that the body of a transaction is executed with +the @code{*store-controller*} variable bound to the store-controller +object creating the transaction. This is important for default +arguments and generally helps more than it hurts, so is an +implementation requirement placed on @code{execute-transaction}. + +If two nested calls to @code{with-transaction} are made successively +in a dynamic context, the data store can create true nested +transactions. The first transaction is passed to the @code{:parent} +argument of the second. The second can choose to just continue the +current transaction (the CLSQL data store policy) or to nest the +transaction (the BDB data store policy). + +@subsection Interleaving Multiple Store Transactions + +Finally, some provision is made for the case where two store +controllers have concurrently active transactions in the same thread. +This feature was created to allow for migration, where a read from one +database happens in one transaction, and while active has to writes to +another data store with a valid transaction. + +The trick is that @code{with-transaction} checks to see if the current +transaction object is the same as the @code{store-controller} object +passed to the @code{:store-controller} argument. If not, a fresh +transaction is started. + +Currently no provision is made for more than two levels of multi-store +nesting as we do not implement a full transaction stack (to avoid +walking the stack on each call to handle this rare case). If a third +transaction is started by the store controller that started the first +transaction, it will have no access to the parent transaction which +may be a significant source of problems for the underlying database. + --- /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/04/21 17:22:35 1.9 +++ /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/04/27 03:14:55 1.10 @@ -6,8 +6,8 @@ @copying Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. -2006 SQL Backend, Copyright @copyright{} 2006 Robert L. Read. -2007 Rewrite, Copyright @copyright{} 2007 Ian Eslick. +Version 0.5, Copyright @copyright{} 2006 Robert L. Read. +Versions 0.6-0.9, Copyright @copyright{} 2006-2007 Ian Eslick and Robert L. Read @quotation Permission is granted to copy, distribute and/or modify this document @@ -21,7 +21,7 @@ @titlepage @title Elephant User Manual @subtitle Elephant version 0.9 -@author Ian Eslick and Ben Lee +@author By Ian Eslick with Robert Read and Ben Lee @c The following two commands @c start the copyright page. --- /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/04/21 17:22:35 1.10 +++ /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/04/27 03:14:55 1.11 @@ -212,7 +212,8 @@ ... => #<BDB-STORE-CONTROLLER> -(open-store '(:CLSQL (:POSTGRESQL "localhost.localdomain" "mydb" "myuser" "")))) +(open-store '(:CLSQL (:POSTGRESQL "localhost.localdomain" + "mydb" "myuser" "")))) ... ASDF loading messages ... --- /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/04/25 02:27:56 1.12 +++ /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/04/27 03:14:55 1.13 @@ -8,21 +8,23 @@ @menu * Store Controllers:: Connecting to a data store. -* Persistent Objects:: Defining persistent classes and creating and manipulating persistent indices. -* Persistent Object Indexing:: Convenient indexing of persistent classes. -* Persistent Sets:: Maintaining persistent collections the easy way. +* Persistent Object API:: Defining persistent classes and creating and manipulating persistent indices. +* Persistent Object Indexing API:: Convenient indexing of persistent classes. +* Persistent Set API:: Maintaining persistent collections the easy way. @c * Query Interfaces:: Finding instances. -* BTrees:: BTrees and indices, a low level persistent data structure. -* Cursors:: Traversing BTrees. -* Transactions:: Transactions. -* Migration and Upgrading:: Migration and upgrading. -* Miscellaneous:: Other functions and data store specific functions +* BTree API:: BTrees and indices, a low level persistent data structure. +* Cursor API:: Traversing BTrees. +* Index Cursor API:: Traversing BTree Indicies. +* Transaction API:: Transaction functions. +* Migration and Upgrading API:: Migration and upgrading. +* Miscellaneous API:: Other functions and data store specific functions @end menu -@node Store Controllers +@node Store Controller API @comment node-name, next, previous, up -@section Store Controllers -@cindex Store Controllers +@section Store Controller API +@cindex Store Controller +@cindex API Store controllers provide the persistent storage for CLOS objects and BTree collections. Any persistent operations must be done in the @@ -45,10 +47,11 @@ @include includes/fun-elephant-root-existsp.texinfo @include includes/fun-elephant-map-root.texinfo -@node Persistent Objects +@node Persistent Object API @comment node-name, next, previous, up -@section Persistent Objects +@section Persistent Object API @cindex Persistent Objects +@cindex API @ref{Class elephant:persistent-metaclass} can be used as the :metaclass argument in a defclass form to create a persistent object. @@ -58,15 +61,17 @@ @include includes/macro-elephant-defpclass.texinfo @include includes/fun-elephant-drop-pobject.texinfo -@node Persistent Object Indexing +@node Persistent Object Indexing API @comment node-name, next, previous, up -@section Persistent Object Indexing +@section Persistent Object Indexing API @cindex Persistent Object Indexing +@cindex API + @subsection Indexed Object Accessors @include includes/fun-elephant-map-class.texinfo -@include includes/fun-elephant-map-class-index.texinfo +@include includes/fun-elephant-map-inverted-index.texinfo @include includes/fun-elephant-get-instances-by-class.texinfo @include includes/fun-elephant-get-instance-by-value.texinfo @@ -93,10 +98,10 @@ @include includes/fun-elephant-add-class-derived-index.texinfo @include includes/fun-elephant-remove-class-derived-index.texinfo -@node Persistent Sets +@node Persistent Set API @comment node-name, next, previous, up -@section Persistent Sets -@cindex Persistent Sets +@section Persistent Set API +@cindex Persistent Set API Persistent sets are a simple persistent collection abstraction. They maintain an unordered collection of objects. Unlike the normal @@ -125,10 +130,12 @@ @c @include includes/fun-elephant-get-query-results.texinfo @c @include includes/fun-elephant-map-class-query.texinfo -@node BTrees +@node BTree API @comment node-name, next, previous, up -@section BTrees -@cindex BTrees +@section BTree API +@cindex BTree +@cindex API + Persistent collections inherit from @ref{Class elephant:persistent-collection} and consist of the @ref{Class elephant:btree}, @ref{Class elephant:indexed-btree} and @@ -142,29 +149,40 @@ @include includes/fun-elephant-setf-get-value.texinfo @include includes/fun-elephant-remove-kv.texinfo @include includes/fun-elephant-existsp.texinfo - +@include includes/fun-elephant-drop-btree.texinfo @include includes/fun-elephant-map-btree.texinfo -@include includes/fun-elephant-map-index.texinfo + +These functions are only defined on indexed btrees. + +@include includes/fun-elephant-make-indexed-btree.texinfo @include includes/fun-elephant-add-index.texinfo @include includes/fun-elephant-get-index.texinfo @include includes/fun-elephant-get-primary-key.texinfo @include includes/fun-elephant-remove-index.texinfo -@include includes/fun-elephant-drop-pset.texinfo -@node Cursors +This function is only valid for indexes. + +@include includes/fun-elephant-map-index.texinfo + +@node Cursor API @comment node-name, next, previous, up -@section Cursors +@section Cursor API @cindex Cursors +@cindex API Cursors are objects of type cursor (@pxref{Class elephant:cursor}) -which provide methods for complex traversals of BTrees. +which provide methods for complex traversals of BTrees. @include includes/macro-elephant-with-btree-cursor.texinfo @include includes/fun-elephant-make-cursor.texinfo @include includes/fun-elephant-cursor-close.texinfo - @include includes/fun-elephant-cursor-duplicate.texinfo +@include includes/fun-elephant-cursor-initialized-p.texinfo + +Each of the following methods return multiple values consisting of +@code{(exists? key value)}. + @include includes/fun-elephant-cursor-current.texinfo @include includes/fun-elephant-cursor-first.texinfo @include includes/fun-elephant-cursor-last.texinfo @@ -177,6 +195,25 @@ @include includes/fun-elephant-cursor-delete.texinfo @include includes/fun-elephant-cursor-put.texinfo +@node Index Cursor API +@comment node-name, next, previous, up +@section Cursor API +@cindex Cursors +@cindex Index +@cindex Indices +@cindex API + +Index cursors are made the same way standard cursors are, with a call +to @code{make-cursor}, except with the index as the argument instead +of a standard btree. In addition to the standard cursor operations, +which provide the direct key and value of a @code{btree-index}, the +following class of ``p'' cursors work on an index and allow you to get +the primary value of the @code{indexed-btree} that the +@code{btree-index} belongs to. + +They each return multiple values @code{(exists? key primary-value +primary-key)}. + @include includes/fun-elephant-cursor-pcurrent.texinfo @include includes/fun-elephant-cursor-pfirst.texinfo @include includes/fun-elephant-cursor-plast.texinfo @@ -194,10 +231,11 @@ @include includes/fun-elephant-cursor-prev-nodup.texinfo @include includes/fun-elephant-cursor-pprev-nodup.texinfo -@node Transactions +@node Transaction API @comment node-name, next, previous, up -@section Transactions -@cindex Transactions +@section Transaction API +@cindex Transaction API +@cindex API @include includes/macro-elephant-with-transaction.texinfo @@ -210,10 +248,11 @@ @include includes/fun-elephant-controller-abort-transaction.texinfo @include includes/fun-elephant-controller-commit-transaction.texinfo -@node Migration and Upgrading +@node Migration and Upgrading API @comment node-name, next, previous, up -@section Migration and Upgrading +@section Migration and Upgrading API @cindex Migration and Upgrading +@cindex API Upgrade is a call to Migrate with checks for compatability. The migrate methods are included here in case you wish to develop a more --- /project/elephant/cvsroot/elephant/doc/scenarios.texinfo 2007/04/25 03:18:13 1.8 +++ /project/elephant/cvsroot/elephant/doc/scenarios.texinfo 2007/04/27 03:14:55 1.9 @@ -82,7 +82,9 @@ (unless *system (let ((appname (get-application-name)) (url (get-system-url))) - (setf *system* (make-instance 'system-object :name appname :url url)))) + (setf *system* (make-instance 'system-object + :name appname + :url url)))) *system*) (sys-object) @@ -102,7 +104,8 @@ (defclass system-object () ((appname :accessor system-appname :initarg :name) (url :accessor system-url :initarg :url) - (laststate :accessor system-laststate :initarg :state :initform 'idle) + (laststate :accessor system-laststate :initarg :state + :initform 'idle) (state :accessor system-state :initarg :state :transient t) (:metaclass persistent-metaclass)) @@ -163,7 +166,8 @@ @lisp (defclass phash () - ((btree :accessor phash-btree :initarg :btree :initform (make-btree)))) + ((btree :accessor phash-btree :initarg :btree + :initform (make-btree)))) (defun make-persistent-hash (name) (let ((btree (get-from-root name))) @@ -423,7 +427,8 @@ :transient t) (touched :accessor snapshot-set-touched :initform (make-array 20 :element-type 'fixnum - :initial-element 0 :fill-pointer t :adjustable t) + :initial-element 0 :fill-pointer t + :adjustable t) :transient t)) (:documentation "Keeps track of a set of standard objects allowing a single snapshot call to update the store @@ -455,7 +460,8 @@ @lisp (defmethod register-object ((object standard-object) (set snapshot-set)) - "Register a standard object. Not recorded until snapshot is called on db" + "Register a standard object. Not recorded until + the snapshot function is called on db" (aif (lookup-cached-id object set) (values object it) (let ((id (incf (snapshot-set-next-id set)))) @@ -506,8 +512,11 @@ (defmethod snapshot ((set snapshot-set)) "Saves all objects in the set (and any objects reachable from the current set of objects) to the persistent store" - (with-transaction (:store-controller (get-con (snapshot-set-index set))) - (loop for (obj . id) in (get-cache-entries (snapshot-set-cache set)) do + (with-transaction (:store-controller (get-con + (snapshot-set-index set))) + (loop for (obj . id) in + (get-cache-entries (snapshot-set-cache set)) + do (save-snapshot-object id obj set)) (collect-untouched set))) @@ -581,11 +590,12 @@ @lisp (defmethod restore ((set snapshot-set)) - "Restores a snapshot by setting the snapshot-set state to the last snapshot. - If this is used during runtime, the user needs to drop all references - to objects and retrieve again from the snapshot set. Also used to initialize - the set state when a set is created, for example pulled from the root of a - store-controller, unless :lazy-load is specified" + "Restores a snapshot by setting the snapshot-set state to the last +snapshot. If this is used during runtime, the user needs to drop all +references to objects and retrieve again from the snapshot set. Also +used to initialize the set state when a set is created, for example +pulled from the root of a store-controller, unless :lazy-load is +specified" (clear-cache set) (map-btree (lambda (id object) (load-snapshot-object id object set)) @@ -597,7 +607,8 @@ (load-proxy-object id object set)) ((hash-table-p object) (load-proxy-hash id object set)) - (t (error "Unrecognized type ~A for id ~A in set ~A" (type-of object) id set))))) + (t (error "Unrecognized type ~A for id ~A in set ~A" + (type-of object) id set))))) @end lisp If an object has a reference object in a slot, then we simply restore @@ -700,7 +711,8 @@ hash table indirections with a little macro: @lisp -(defmacro def-snapshot-reference-wrapper (accessor-name (source-classname target-classname hashname uid)) +(defmacro def-snapshot-wrapper + (accessor-name (source-classname target-classname hashname uid)) (with-gensysms (obj key ref) `(progn (defmethod ,accessorname :around ((,obj ,source-classname)) @@ -723,7 +735,8 @@ ((pointer-to-island2 :accessor child :initform nil) (pointer-to-island1 :accessor neighbor :initform nil))) -(def-snapshot-reference-wrapper neighbor (island2 island1 *island1-hash* *unique-id*)) +(def-snapshot-wrapper neighbor + (island2 island1 *island1-hash* *unique-id*)) @end lisp Of course this doesn't work for multi-threaded environments, or for @@ -815,7 +828,7 @@ relationships such as ``PartOf'', ``DesireOf'' and ``EffectOf'' between English phrases. -Elephant's persistence capabilities is used to keep full records of +Elephant's persistence capability is used to keep full records of all source material, extracted relationships and search queries so that it is always possible to trace the source of a learned relation and to avoid repeated queries to web search engines. Conceptminer @@ -828,13 +841,14 @@ @itemize @item Bulk storage of post-processed web data: Elephant was used to -store hundreds of thousands of processed web pages as strings and index -the queries and resulting metadata. -@item Derived index: a custom string hash function was used to populate -a class derived index, allowing fast identification of pages from their URL. +store hundreds of thousands of processed web pages as strings, associate +pages with queries and store related metadata. +@item Derived index: a custom string hash function over URLs was used to populate +a class derived index, allowing fast identification of pages from +their URL without requiring expensive eql comparisons. @item Inverted document index: a (not terribly efficient) data structure that efficiently maps words to documents allowing pages to indexed by -the words contained in them. Allows for eash phrase and conjunction searches. +the words contained in them. Allowed for phrase and conjunction searches. @item User association data structure: a data structure based on oids that supports general one-to-many mappings between classes. Had a custom migrate method to support migration of associations. Supplanted by persistent @@ -845,13 +859,13 @@ architecture to cover in-memory lisp operations. @code{PCOMP} (Process Components) is a framework for constructing and managing simple, dataflow-style multi-threaded applications in Common Lisp. The goal -is to simplify the process sufficiently so that the ordinary user does -not hurt themselves easily. To this end, the model provides for safe, -asynchronous communications among a set of components which may be -scheduled together in a single process or communicate across separate -threads (and potentially processes). Components are packaged into a -system inside a Container object which schedules execution and -mediates communications. +is to simplify the process sufficiently so that the ordinary user can +hide from many of the details associated with aborting transactions. +To this end, the model provides for safe, asynchronous communications +among a set of components which may be scheduled together in a single +process or communicate across separate threads (and potentially +processes). Components are packaged into a system inside a Container +object which schedules execution and mediates communications. Communications between components can be in a dataflow style or using messages. Each component has a single port for receiving incoming data items. These items, if access --- /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/04/25 02:27:56 1.18 +++ /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/04/27 03:14:55 1.19 @@ -63,19 +63,22 @@ @item @strong{Store controller:} the interface between lisp and a data store. Most operations require or accept a store controller, or a default store controller stored in @code{*store-controller*} to function. +@item @strong{Persistent Sets:} A simple persistent collection is provided +which allows the creation of persistent sets. @item @strong{BTrees:} Elephant provides a persistent key-value abstraction based on the BTree data structure. Values can be written to or read from a BTree and are stored in a sorted order. -@item @strong{Values:} most lisp values, including standard objects, arrays, etc +@item @strong{Stored values:} most lisp values, including standard objects, arrays, etc can be used as either key or value in a persistent BTree. @item @strong{Persistent objects:} An object where most slot values are stored in the data store and are written to or retrieved from disk on slot -accesses. Any value that can be written to an index can be written to -a persistent slot. -@item @strong{Transactions:} a dynamic context for executing operations on persistent -objects or BTrees that ensures that a set of changes is made atomically. -@item @strong{BTree indices:} A BTree index is a BTree that stores -an alternative ordering of the elements in a reference BTree. +accesses. Storing a persistent object stores only a reference, +allowing for object identity. +@item @strong{Object indexing:} The ability to lookup and sort objects by their slot +values rather than by explicit inclusion in a collection. +@item @strong{Transactions:} a dynamic context for executing operations on objects +or collections such that the side effects exhibit the ACID (atomicity, consistency, +isolation and durability) properties of database. @end itemize There are a set of more advanced concepts you will learn about later, @@ -112,7 +115,8 @@ @lisp (open-store '(:CLSQL (:SQLITE "/users/me/db/sqlite.db"))) -(open-store '(:CLSQL (:POSTGRESQL "localhost.localdomain" "mydb" "myuser" "")))) +(open-store '(:CLSQL (:POSTGRESQL "localhost.localdomain" + "mydb" "myuser" "")))) @end lisp We use Berkeley DB as our example backend. To open a BDB @@ -197,7 +201,7 @@ => #<DB-BDB::BDB-BTREE @ #x10e86042> @end lisp -It is an instance of a class "btree"; @pxref{Using BTrees}. +It is an instance of a class "btree"; @pxref{Persistent BTrees}. @node Serialization @comment node-name, next, previous, up @@ -255,7 +259,7 @@ This will affect all aggregate types: objects, conses, hash-tables, et cetera. (You can of course manually re-store the cons.) In this sense elephant does not automatically provide persistent collections. If you -want to persist every access, you have to use BTrees (@pxref{Using BTrees}). +want to persist every access, you have to use BTrees (@pxref{Persistent BTrees}). @item @strong{Serialization and deserialization can be costly}. While serialization is pretty fast, but it is still expensive to store large @@ -504,7 +508,8 @@ @lisp (defpclass person () ((name :accessor person-name :initarg :name)) - ((friends :accessor person-friends :initarg :friends :initform (make-pset)))) + ((friends :accessor person-friends :initarg :friends + :initform (make-pset)))) (defmethod add-friend ((me person) (them person)) (insert-item them (person-friends me))) @@ -522,7 +527,8 @@ @lisp (defpclass person () ((name :accessor person-name :initarg :name)) - ((friends :accessor person-friends-set :initarg :friends :initform (make-pset)))) + ((friends :accessor person-friends-set :initarg :friends + :initform (make-pset)))) (defmethod person-friends ((me person)) (pset-list (person-friends-set me))) @@ -609,7 +615,8 @@ @lisp (map-btree (lambda (k v) (format t "name: ~A utime: ~A~%" k - (subseq (multiple-value-list (decode-universal-time v)) 3 6))) + (subseq (multiple-value-list + (decode-universal-time v)) 3 6))) *friends-birthdays*) "Andrew" "Ben" @@ -675,11 +682,15 @@ @lisp (defun print-friend (friend) - (format t " name: ~A birthdate: ~A~%" (name friend) (birthday friend))) + (format t " name: ~A birthdate: ~A~%" + (name friend) (birthday friend))) -(make-instance 'friend :name "Carlos" :birthday (encode-date '(1 1 1972))) -(make-instance 'friend :name "Adriana" :birthday (encode-date '(24 4 1980))) -(make-instance 'friend :name "Zaid" :birthday (encode-date '(14 8 1976))) +(make-instance 'friend :name "Carlos" + :birthday (encode-date '(1 1 1972))) +(make-instance 'friend :name "Adriana" + :birthday (encode-date '(24 4 1980))) +(make-instance 'friend :name "Zaid" + :birthday (encode-date '(14 8 1976))) (get-instances-by-class 'friends) => (#<Carlos> #<Adriana> #<Zaid>) @@ -782,7 +793,8 @@ name: Carlos birthdate: (1 1 1972) => NIL -(map-class-index #'print-friend 'friend 'name :start "Adam" :end "Devin") +(map-class-index #'print-friend 'friend 'name + :start "Adam" :end "Devin") name: Adriana birthdate: (24 4 1980) name: Carlos birthdate: (1 1 1972) => NIL @@ -817,7 +829,7 @@ specification differs. This same facility is also available for your own use. For more -information @pxref{Secondary Indices}. +information @pxref{BTree Indexing}. @node Using Transactions @@ -1195,12 +1207,12 @@ and iterating of duplicate or unique values. @item @strong{Using the Map Operators} Mapping operators can be very efficient if properly utilized. -@item @strong{Using Multiple Stores} - Multiple store controllers can be open simultaneously. However it -does make the code more complex and you need to be careful about how -you use them to avoid crashes and other unpleasant side effects. -@item @strong{Custom Transaction Architecture} - You can implement your own version of with-transaction using the +@item @strong{Using Multiple Stores} Multiple store controllers can +be open simultaneously. However it does make the code more complex +and you need to be careful about how you use them to avoid crashes +and other unpleasant side effects. +@item @strong{Custom Transaction Architecture} You can implement your +own version of @code{with-transaction} using the underlying controller methods for starting, aborting and committing transactions. You had better know what you are doing, however! @item @strong{Handling Errors and Conditions} --- /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/04/26 03:10:13 1.18 +++ /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/04/27 03:14:55 1.19 @@ -10,8 +10,8 @@ * Serialization details:: The devil hides in the details. * Persistent Classes and Objects:: All the dirt on persistent objects. * Class Indices:: In-depth discussion about indexing objects. -* Persistent Sets:: Using the persistent set. -* Persistent BTrees:: Using the native btree. +* Persistent Sets:: Using the persistent set collection. +* Persistent BTrees:: Using the native btree collection. * BTree Cursors:: Low-level access to BTrees. * BTree Indexing:: Alternative ways to reference objects in btrees. * Index Cursors:: Low-level access to BTree indices. @@ -172,9 +172,10 @@ @end lisp This will affect all aggregate types: objects, conses, hash-tables, et -cetera. (You can of course manually re-store the cons.) In this sense -elephant does not automatically provide persistent collections. If you -want to persist every access, you have to use BTrees (@pxref{Using BTrees}). +cetera. (You can of course manually re-store the cons.) In this +sense elephant does not automatically provide persistent collections. +If you want to persist every access, you have to use Persistent Sets +(@pxref{Persistent Sets}) or BTrees (@pxref{Persistent BTrees}). @item @strong{Serialization and deserialization can be costly}. While serialization is pretty fast, but it is still expensive to store large @@ -382,8 +383,10 @@ @lisp (defpclass my-pclass () ((pslot1 :accessor pslot1 :initarg :pslot1 :initform 'one) - (pslot2 :accessor pslot2 :initarg :pslot2 :initform 'two :persistent t) - (tslot1 :accessor tslot1 :initarg :tslot1 :initform 'three :transient t))) + (pslot2 :accessor pslot2 :initarg :pslot2 :initform 'two + :persistent t) + (tslot1 :accessor tslot1 :initarg :tslot1 :initform 'three + :transient t))) @end lisp In the definition above the class @code{my-pclass} is an instance of @@ -706,8 +709,9 @@ Because the instance is guaranteed to be resident, the operation has none of the resident/non-resident conflicts above. -Changing a persistent instance to a non-persistent class is not -allowed and will result in an error. +Change class cannot convert between persistent and non-persistent classes and will +flag an error if you try to do so. @emph{(Note: this could be implemented in the +future if users request it)} @node Class Indices @comment node-name, next, previous, up @@ -860,7 +864,6 @@ database for reuse. The oid used by the btree, however, will not be recovered. -@c *** FINISH *** @node BTree Cursors @comment node-name, next, previous, up @section BTree Cursors @@ -869,27 +872,139 @@ database, you can also traverse the BTree structure one key-value pair at a time. -Creating cursors (inside transactions) -Duplicating cursors +Cursors must be created in the context of an active transaction +(i.e. a @code{with/ensure-transaction} body). A cursor is made +through a call to the @code{make-cursor} method of the BTree you wish +to traverse. + +An existing cursors can also be duplicated within the same transaction +by calling @code{cursor-duplicate} which avoids the overhead of +setting a second cursor to the same location. -Initializing cursors to location -- first, last, set -Traversing -- next, prev, (set) -Side effects -- delete -- put -- Can also use setf get-value while traversing +Cursors can be in two states: initialized and uninitialized. +@ref{Cursor API} for details. -Large cursor operations (locks, resources?) +To initialize a cursor, you have to use one of the initializing +functions to select a key-value pair in the btree. + +@itemize +@item @code{cursor-first} and @code{cursor-last}: initialize the +cursor to the first and last element of the btree, respectively. +@item @code{cursor-set} and @code{cursor-set-range}: Sets the cursor +to the first key-pair values according to the specified key. If the +set fails, the cursor will remain uninitialized. The ranged set will +set it to the first key-value pair where the key is equal to or +greater than the key argument. +@end itemize + +A valid cursor will return multiple values: @code{(exists? key +value)}. The first argument tells whether or not the cursor is +initialized and pointing at a proper value. The second two arguments +are self-explanatory. + +@code{cursor-current} returns the current state of the cursor, nil if +it is uninitialized. + +Once a cursor is properly initialized, it can be incremented or +decremented, a simple constant-time operation on BTrees. + +@code{cursor-next} and @{cursor-prev} move the cursor a single step +forward or back across the sorted key-value pairs. @code{cursor-next} +moves in ascending order, @code{cursor-prev} in descending order. + +Finally cursors can be used for side effects on the current key-value +pair. The function @code{cursor-put} replaces the value (but does not +increment the current value) and @code{cursor-delete} deletes the +key-value pair and become uninitialized. It is a valid operation to +use the @code{(setf get-value)} method while the cursor is active to +change the value at the current cursor. + +If cursors take place within a transaction, what happens when +traversing a very large BTree? This depends on the data store policy +regarding whether a cursor read locks its entire btree (or the subset +that is being iterated over) or allows changes to any pairs its +transaction has not changed. See your data store documenation for +details. -@c *** FINISH *** @node BTree Indexing @comment node-name, next, previous, up @section BTree Indexing +One powerful feature of Elephant is the ability to add indexes to +BTrees. An indexed btree is a subclass of the standard @code{btree} +called @code{indexed-btree}. The indexed btree maintain a set of +indices (instances of @code{btree-index}) which provide alternative +ways of indexing into the values of the main btree. + +Each index is itself a btree, but with the property that its values +are matched to the keys of the main btree. That is if you have a btree +with key-value pairs: -@c *** FINISH *** +@lisp +("henry" . #<name: Henry, age: 45>) +("larry" . #<name: Larry, age: 29>) +@end lisp + +You can define an index that is populated by the age of the person object: + +@lisp +(29 . "Larry") +(45 . "Henry") +@end lisp + +Now when you call @code{(get-value 29 index)} you get back +@code{#<name: Larry, age: 29>}! Note also that these new pairs are +ordered by age, the opposite of the alphabetic ordering of the names +in the first two pairs. If you read through the tutorial, you may +have guessed by now that this is the mechanism used to implement the +class indexing capabilities previously described. + +An index is created by using the @code{add-index} function. This +function takes the @code{indexed-btree} you wish to index, an +index-name for later retrieval, a key-form which dictates how +the index populates it's keys as a function of the main btree's +keys and values. + +A simple, contrived example is shown in the figure below: + +@center @image{IndexedBtrees1,,4.0in,[BTree Index Diagram],.png} + +Here we have a primary, indexed btree with a set of keys and values +represented by symbols. We'll declare the function @code{val} to take +a value symbol and extract it's number. The key-form in the +@code{mod5 * 2} index is: + +@lisp +(lambda (k v) + (if (= 0 (mod (val v) 5)) + (values t (* 2 (val v))) + (values nil nil))) +@end lisp + +When a key-value pair is written to the primary btree, the index is +automatically updated through a call to the key-form. If the key-form +above is called with @code{key1} and @code{value1}, @code{val} will +return 1 which fails the if test. The second values statement, +@code{(values nil nil)} indicates that this pair is not to be indexed. +If I pass @code{key5} and @code{value5} to this same key form, I get +back 10 as the @code{(val 'value5)} is 5 and @code{(= 0 (mod 5 5))} so +the form returns @code{(values t 10)} meaning the index should add an +index entry of 10 (@code{(* 2 5)}) associated with the key value +@code{key5}. + +So, of course, making the call @code{(get-value 10 index-mod5)} will +return @code{value5}. + +The second index in our little example calculates the number of bits +in all odd numbered values. This illustrates an important property of +the @code{btree-index}: it allows duplicate keys. Standard +@code{btree} and @code{indexed-btree} classes are not allowed to have +duplicate elements. The odd index allows us to ask simple questions +like: ``what are all the odd values with ids that fit into 4 bits?''. + +To extract this set, we have to use cursor functions specifically +designed for the index that iterate over duplicate values. + @node Index Cursors @comment node-name, next, previous, up @section Index Cursors @@ -897,19 +1012,29 @@ Index cursors are just like BTree cursors except you can get the main BTree value instead of the index value. There are also a parallel set of operations such as @code{cursor-pnext} instead of -@code{cursor-next} which returns @code{exists}, @code{key}, @code{primary-btree-value} -and @code{index-value = primary-btree-key}. - -Operations that have the same behavior, but return primary btree values and keys are: +@code{cursor-next} which returns @code{exists}, @code{key}, +@code{primary-btree-value} and @code{index-value = primary-btree-key}. -@c need table here +Operations that have the same behavior, but return primary btree +values and keys are: -@itemize -@item cursor-first -> cursor-pfirst, cursor-last -> cursor-plast -@item cursor-current -> cursor-pcurrent -@item cursor-next -> cursor-pnext, cursor-prev -> cursor-pprev -@item cursor-set-range -@end itemize +@table @samp +@item @code{cursor-first} +@code{cursor-pfirst} +@samp{@code{cursor-first}} +@item @code{cursor-last } +@code{cursor-plast} +@item @code{cursor-current } +@code{cursor-pcurrent} +@item @code{cursor-next } +@code{cursor-pnext} +@item @code{cursor-prev} +@code{cursor-pprev} +@item @code{cursor-set} +@code{cursor-pset} +@item @code{cursor-set-range} +@code{cursor-set-prange} +@end table The big difference between btree cursors and index cursors is that indices can have duplicate key values. This means we have to choose @@ -1256,6 +1381,10 @@ @item @strong{@code{:txn-sync}} This is the default behavior and specifies that the transaction log of the current transaction is flushed to disk before the transaction commit routine returns. This provides full ACID compliance. +@item @strong{@code{:transaction}} This argument is for advanced use. It tells + the Berkeley DB transaction subsystem the transaction it should use rather +than to create a new one. The @code{:parent} argument provides a parent transaction +that can result in a true nested transaction. @end itemize @subsection Special Commands @@ -1357,7 +1486,9 @@ @comment node-name, next, previous, up @section Native Lisp Data Store -The native lisp data store is vaporware at this time. +The native lisp data store is unimplemented. It is tentatively +planned for a 1.1 release sometime in the distant future. Yes, +this is a deliberatively vague declaration. @c @node Querying persistent instances @c @comment node-name, next, previous, up

1 0