March 2007 - elephant-cvs@common-lisp.net

[elephant-cvs] CVS elephant/tests
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/tests In directory clnet:/tmp/cvs-serv7809/tests Added Files: stress-test.lisp Log Message: Some working files that were missing --- /project/elephant/cvsroot/elephant/tests/stress-test.lisp 2007/03/30 15:03:46 NONE +++ /project/elephant/cvsroot/elephant/tests/stress-test.lisp 2007/03/30 15:03:46 1.1 (in-package :elephant-tests) (defparameter *spec* '(:bdb "/Users/eslick/Work/db/test")) (defparameter *names* '("David" "Jim" "Peter" "Thomas" "Arthur" "Jans" "Klaus" "James" "Martin")) (defclass person () ((name :initform (elt *names* (random (length *names*))) :accessor name :index t) ;; Actually the index t shouldn't be needed, but since elephant ;; sometimes complained that "person is not an index class", I try if this fixes it. (age :initform (random 100) :accessor age :index t) ;; (made-by :initform (elephant-utils::ele-thread-hash-key)) (updated-by :initform nil :accessor updated-by)) (:metaclass elephant:persistent-metaclass)) (defparameter *nr-persons* 10000) ;; Should be 10000, but for me elephant can't allocate memory after 3000. ;; I think the problem it is becuase the number of locks (999) is = max 1000. see db_stat -e (defparameter +age+ 50) ;; I have tried different places for with-transaction below (defun make-persons (nr-objects &optional (batch-size 500)) (loop for i from 1 to (/ nr-objects batch-size) do (elephant:with-transaction () (loop for j from 1 to batch-size do (let ((person (make-instance 'person))) (when (zerop (mod (+ (* i batch-size) j) 1000)) (format t "~D ~a " (+ (* i batch-size) j) (name person)))))))) (defun ensure-clean-store () t) ;; (let ((dir (cl-fad:pathname-as-directory (second *spec*)))) ;; (when (cl-fad:directory-exists-p dir) ;; (cl-fad:delete-directory-and-files dir)) ;; (ensure-directories-exist dir))) (defun my-test-create () (ensure-clean-store) (elephant:with-open-store (*spec*) (make-persons *nr-persons*))) (defun subsets (size list) (let ((subsets nil)) (loop for elt in list for i from 0 do (when (= 0 (mod i size)) (setf (car subsets) (nreverse (car subsets))) (push nil subsets)) (push elt (car subsets))) (setf (car subsets) (nreverse (car subsets))) (nreverse subsets))) (defmacro do-subsets ((subset subset-size list) &body body) `(loop for ,subset in (subsets ,subset-size ,list) do ,@body)) (defun my-test-update (&key (new-age 27)) "Test updating all persons by changing their age." (elephant:with-open-store (*spec*) (do-subsets (subset 500 (elephant:get-instances-by-class 'person)) (format t "Doing subset~%") (elephant:with-transaction () (mapcar #'(lambda (person) (setf (age person) new-age)) subset))))) (defun my-test-load () "Test loading all persons by computing their average age." (let ((nr-persons 0) (total-age 0) (show-first nil)) (elephant:with-open-store (*spec*) (elephant:with-transaction () (mapcar #'(lambda (person) (incf nr-persons) (print nr-persons) (when (and show-first (> show-first)) (format t "Sample person ~a~%F" show-first) (describe person) (decf show-first)) (incf total-age (age person))) (elephant:get-instances-by-class 'person)))) (values (coerce (/ total-age nr-persons) 'float) nr-persons total-age))) (defun check-basic-setup () (my-test-update :new-age +age+) (multiple-value-bind (average nr-persons) (my-test-load) (assert (= +age+ average)) (assert (= nr-persons *nr-persons*))))

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv4653/src/elephant Modified Files: backend.lisp package.lisp Log Message: Add exported symbols that were missing from backend.lisp --- /project/elephant/cvsroot/elephant/src/elephant/backend.lisp 2007/03/30 14:34:35 1.12 +++ /project/elephant/cvsroot/elephant/src/elephant/backend.lisp 2007/03/30 14:55:54 1.13 @@ -19,57 +19,21 @@ (in-package :cl-user) -(defpackage :elephant-backend +(defmacro defpackage-import-exported (name source-package &rest args) + "Define an export list, a source package and this macro will automatically + import from that package the exported symbol names." + (let* ((exports (find :export args :key #'car)) + (imports `(:import-from ,source-package ,@(cdr exports)))) + `(defpackage ,name + ,@(append args (list imports))))) + +(defpackage-import-exported :elephant-backend :elephant (:documentation "Backends should use this to get access to internal symbols of elephant that importers of elephant shouldn't see. Backends should also import elephant to get use-api generic function symbols, classes and globals") (:use #:elephant) - (:import-from #:elephant - ;; Variables - #:*dbconnection-spec* - #:connection-is-indeed-open - - ;; Persistent objects - #:oid #:get-con - #:next-oid - #:persistent-slot-writer - #:persistent-slot-reader - #:persistent-slot-boundp - #:persistent-slot-makunbound - - ;; Controllers - #:*elephant-code-version* - #:open-controller - #:close-controller - #:database-version - #:controller-spec - #:controller-serialize - #:controller-deserialize - #:root #:spec #:class-root - ;; Serialization - #:deserialize-from-base64-string - #:serialize-to-base64-string - ;; Cursor accessors - #:cursor-btree - #:cursor-oid - #:cursor-initialized-p - ;; Transactions - #:*current-transaction* - #:make-transaction-record - #:transaction-store - #:transaction-object - ;; Registration - #:register-backend-con-init - #:lookup-backend-con-init - ;; Misc - #:slot-definition-name - #:slots-and-values - #:struct-slots-and-values - #:remove-indexed-element-and-adjust - ) (:export ;; Variables - #:*cachesize* #:*dbconnection-spec* #:connection-is-indeed-open @@ -87,25 +51,37 @@ #:close-controller #:database-version #:controller-spec - #:controller-version + #:controller-serializer-version #:controller-serialize #:controller-deserialize #:root #:spec #:class-root - ;; Serialization + + ;; Serializer tools/api's + #:serialize #:deserialize #:deserialize-from-base64-string #:serialize-to-base64-string + #:initialize-serializer + #:serialize-database-version-key + #:serialize-database-version-value + #:deserialize-database-version-value + ;; Cursor accessors #:cursor-btree #:cursor-oid #:cursor-initialized-p + ;; Transactions #:*current-transaction* #:make-transaction-record #:transaction-store #:transaction-object + #:execute-transaction + ;; Registration #:register-backend-con-init #:lookup-backend-con-init + #:get-user-configuration-parameter + ;; Misc #:slot-definition-name #:slots-and-values --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/03/30 14:34:35 1.26 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/03/30 14:55:54 1.27 @@ -206,9 +206,10 @@ #:persistent #:persistent-object #:persistent-metaclass #:defpclass #:persistent-collection #:drop-pobject - #:btree #:make-btree - #:get-value #:remove-kv #:existp - #:indexed-btree #:make-indexed-btree + #:btree #:build-btree + #:get-value #:remove-kv #:existsp + #:indexed-btree #:build-indexed-btree + #:btree-index #:add-index #:get-index #:remove-index #:map-indices #:get-primary-key #:primary #:key-form #:key-fn #:with-btree-cursor #:map-btree #:map-index

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv4653/src/db-bdb Modified Files: package.lisp Log Message: Add exported symbols that were missing from backend.lisp --- /project/elephant/cvsroot/elephant/src/db-bdb/package.lisp 2007/02/18 10:58:58 1.7 +++ /project/elephant/cvsroot/elephant/src/db-bdb/package.lisp 2007/03/30 14:55:54 1.8 @@ -30,7 +30,7 @@ Elephant, but with some magic for Elephant. In general there is a 1-1 mapping from functions here and functions in Berkeley DB, so refer to their documentation for details.") - (:use common-lisp uffi elephant-memutil elephant-backend elephant-utils elephant) + (:use common-lisp uffi elephant-memutil elephant elephant-backend elephant-utils) #+cmu (:use alien) #+sbcl

1 0

[elephant-cvs] CVS elephant/tests
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/tests In directory clnet:/tmp/cvs-serv31032/tests Modified Files: elephant-tests.lisp Log Message: Significant documentation string and documentation edits towards 0.6.1 manual. Clean up packages so elephant exports user visible symbols and backend exports backend-relevant symbols. Change required fix in serializer packages also. Added :elephant-user package. --- /project/elephant/cvsroot/elephant/tests/elephant-tests.lisp 2007/03/12 01:32:06 1.30 +++ /project/elephant/cvsroot/elephant/tests/elephant-tests.lisp 2007/03/30 14:34:35 1.31 @@ -19,7 +19,7 @@ (defpackage elephant-tests (:nicknames :ele-tests) (:use :common-lisp :elephant :regression-test) - (:import-from :ele + (:import-from :elephant with-buffer-streams serialize deserialize)

1 0

[elephant-cvs] CVS elephant/doc
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/doc In directory clnet:/tmp/cvs-serv31032/doc Modified Files: copying.texinfo data-store-reference.texinfo elephant-design.texinfo elephant.texinfo installation.texinfo make-ref.lisp reference.texinfo tutorial.texinfo user-guide.texinfo Log Message: Significant documentation string and documentation edits towards 0.6.1 manual. Clean up packages so elephant exports user visible symbols and backend exports backend-relevant symbols. Change required fix in serializer packages also. Added :elephant-user package. --- /project/elephant/cvsroot/elephant/doc/copying.texinfo 2007/03/24 12:16:02 1.3 +++ /project/elephant/cvsroot/elephant/doc/copying.texinfo 2007/03/30 14:34:34 1.4 @@ -1,38 +1,39 @@ @c -*-texinfo-*- -@node Copying +@node Copyright and License @comment node-name, next, previous, up -@chapter Copying -@cindex Copying +@chapter Copyright and License +@cindex Copyright and License @cindex License -@quotation -@b{Elephant}: an object-oriented database for Common Lisp. +@section Elephant Licensing + +@b{Elephant}: a persistent metaprotocol and object-oriented database +for Common Lisp. Homepage: @uref{http://www.common-lisp.net/project/elephant} +@quotation Elephant users are granted the rights to distribute and use this software as governed by the terms of the Lisp Lesser GNU Public License @uref{http://opensource.franz.com/preamble.html}, also known as the LLGPL. +@end quotation Copyrights include: +@quotation Copyright (c) 2004 by Andrew Blumberg and Ben Lee -Copyright (c) 2006-2007 by Ian Eslick - Copyright (c) 2005-2007 by Robert L. Read +Copyright (c) 2006-2007 by Ian Eslick +@end quotation -Portions of this program (namely the C unicode string -sorter) are derived from IBM's @b{ICU}: - -@uref{http://oss.software.ibm.com/icu/} - -whose copyright and license follows the GPL below. - - +Portions of this program (namely the C unicode string sorter) are +derived from IBM's @b{ICU}: @uref{http://oss.software.ibm.com/icu/, +ICU Website} whose copyright and license follows below. +@quotation ICU License - ICU 1.8.1 and later COPYRIGHT AND PERMISSION NOTICE @@ -72,5 +73,32 @@ All trademarks and registered trademarks mentioned herein are the property of their respective owners. - @end quotation + +@section Data Store Licensing Considerations + +The Berkeley DB data store is based on the Berkeley DB C library, now +owned by Oracle, but available as GPL'ed software. It is important to +understand that applications using Berkeley DB must also be GPL'ed +unless you negotiate a commercial license from Oracle. In most +interpretations of the license, this includes a requirement to make +code available for the entirety of any publicly visible website that +is based on Berkeley DB. See + +@uref{http://www.oracle.com/@/technology/@/software/@/products/@/berkeley-db/@/htdocs/bdboslicense.html}. + +The CL-SQL backend, depending on which SQL engine you use, may not +carry this restriction and you can easily migrate data between the +two. Since the Berkeley DB store is 4-5x faster than SQL, it may make +sense to develop under BDB and transition to SQL after you've tuned +the performance of the application. Licenses for various SQL engines +can be found at: + +@itemize +@item SQLite: Public Domain, see @uref{http://www.sqlite.org/copyright.html, the SQLite license page} +@item Postgresql: BSD License, see @uref{http://www.postgresql.org/about/licence, the Postgresql license page} +@item MySQL: Dual licensing (similar to BDB), see @uref{http://www.mysql.com/company/legal/licensing/, the MySQL license page} +@end itemize + + + --- /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/03/24 13:55:15 1.1 +++ /project/elephant/cvsroot/elephant/doc/data-store-reference.texinfo 2007/03/30 14:34:34 1.2 @@ -7,24 +7,34 @@ @cindex Data Store @cindex API Reference -These are the functions that need to be overridden to implement -support for a data store backend. Included are the exported elephant -functions that need methods defined on them. Some functions here are -utilities from the main elephant package that support store -implementations. Migration, class indices and query interfaces are +This reference includes functions that need to be overridden, classes +inherited from or other action taken to implement support for a new +data store backend. Included are the exported elephant functions that +need methods defined on them as well as the backend-only functions +exported in backends.lisp. Some functions here are utilities from the +main elephant package that support store implementations, but are not +required. Migration, class indices and query interfaces are implemented on top of the store API and require no special support by implementors. +Because the number of backend implementors is small, this is a minimal +documentation set intended to serve as an initial guide and a +reference. However, it is anticipated that some interaction will be +needed with the developers to properly harden a datastore for release. + +The sections each contain a short guide and a list of functions +relevant to them. + @menu -* Registration:: Register the backend to parse controller specifications +* Registration:: Register the backend to parse controller specifications. * Store Controllers:: Subclassing the store controller. +* Handling Serialization:: Available facilities for serializing objects. +* C Utilities:: Writing primitive C types. * Slot access:: Support for metaprotocol slot access. * Collections:: BTrees and indices. * Cursors:: Traversing BTrees. * Transactions:: Transaction implementation. * Multithreading:: Multithreading considerations. -* Serialization:: Facilities for serializing objects. -* C Utilities:: Writing primitive C types. * Foreign libraries:: Using UFFI and ASDF to build or link foreign libraries @end menu @@ -33,8 +43,17 @@ @section Registration @cindex Registration -@include includes/fun-elephant-register-backend-con-init.texinfo -@include includes/fun-elephant-lookup-backend-con-init.texinfo +Elephant looks at the first element of the specification list to +determine which backend code base to use. The master table for this +information is @code{*elephant-backends*} in elephant/controller.lisp. +This will need to be augmented for every backend with the +specification keyword tag to be used (such as @code{:BDB} or +@code{:CLSQL}) and the required asdf dependencies. + +In addition, the backend source should use an eval-when statement to +call the following function: + +@include includes/fun-elephant-backend-register-backend-con-init.texinfo @node Store Controllers @comment node-name, next, previous, up @@ -44,25 +63,22 @@ Subclass store-controller and implement store and close controller which are called by open-store and close-store respectively. -@include includes/fun-elephant-store-controller.texinfo +@include includes/class-elephant-backend-store-controller.texinfo @include includes/fun-elephant-backend-open-controller.texinfo @include includes/fun-elephant-backend-close-controller.texinfo -The slots for these accessors must be initialized. + @include includes/fun-elephant-backend-database-version.texinfo -@include includes/fun-elephant-backend-controller-serialize.texinfo -@include includes/fun-elephant-backend-controller-deserialize.texinfo -@include includes/fun-elephant-backend-root.texinfo -@include includes/fun-elephant-backend-class-root.texinfo These functions are important utilities for implementing store-controllers. -@include includes/fun-elephant-backend-oid.texinfo @include includes/fun-elephant-backend-get-con.texinfo +@include includes/fun-elephant-backend-oid.texinfo @include includes/fun-elephant-backend-next-oid.texinfo @include includes/fun-elephant-backend-connection-is-indeed-open.texinfo +@include includes/fun-elephant-get-user-configuration-parameter.texinfo @node Slot Access @comment node-name, next, previous, up @@ -82,17 +98,37 @@ @section Collections @cindex Collections -@c #:btree #:btree-index #:indexed-btree -@c #:build-indexed-btree #:build-btree #:existsp -@c #:map-indices +To support collections, the data store must subclass the following +classes. +@include includes/class-elephant-btree.texinfo.texinfo +@include includes/class-elephant-btree-index.texinfo +@include includes/class-elephant-indexed-btree.texinfo + +To create the backend-appropriate type of btree, the backend +implements these methods aginst their store-controller. + +@include includes/fun-elephant-build-btree.texinfo +@include includes/fun-elephant-build-indexed-btree.texinfo + +And every btree needs accessors, these must be implemented for btree, +indexed-btree and btree-index. + +@include includes/fun-elephant-get-value.texinfo +@include includes/fun-elephant-setf-get-value.texinfo +@include includes/fun-elephant-existsp.texinfo +@include includes/fun-elephant-remove-kv.texinfo + +@include includes/fun-elephant-map-indices.texinfo +@include includes/fun-elephant-get-index.texinfo +@include includes/fun-elephant-remove-index.texinfo @node Cursors @comment node-name, next, previous, up @section Cursors @cindex Cursors -@c #:cursor +@include includes/class-cursor.texinfo @c #:cursor-btree @c #:cursor-oid @c #:cursor-initialized-p @@ -106,19 +142,23 @@ @c #:make-transaction-record @c #:transaction-store @c #:transaction-object + @c #:execute-transaction @c #:controller-start-transaction @c #:controller-commit-transaction @c #:controller-abort-transaction -@node Multithreading +@node Multithreading Considerations @comment node-name, next, previous, up -@section Multithreading +@section Multithreading Considerations @cindex Multithreading -@node Serialization +@c utils locks +@c utils thread-vars + +@node Handling Serialization @comment node-name, next, previous, up -@section Serialization +@section Handling Serialization @cindex Serialization @c #:deserialize #:serialize --- /project/elephant/cvsroot/elephant/doc/elephant-design.texinfo 2007/03/24 13:55:15 1.1 +++ /project/elephant/cvsroot/elephant/doc/elephant-design.texinfo 2007/03/30 14:34:34 1.2 @@ -1,5 +1,16 @@ -Debugger entered--Lisp error: (void-variable Design) - eval(Design) - eval-last-sexp-1(nil) - eval-last-sexp(nil) - call-interactively(eval-last-sexp) + +@node Elephant Design +@comment node-name, next, previous, up +@section Elephant Design +@cindex design + +When the main elephant @code{open-store} function is called, it calls +@code{get-controller} which grabs an existing store controller if the +spec is identical, or builds a new controller. Building the +controller requires loading any dependencies via asdf, calling a +backend initialization function (if it is the first instance of that +backend being created), and then calling an initialization function +that returns a @code{store-controller} subclass instance specific to +that backend. + +Elephant than calls open-controller --- /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/03/24 12:16:02 1.5 +++ /project/elephant/cvsroot/elephant/doc/elephant.texinfo 2007/03/30 14:34:34 1.6 @@ -5,8 +5,9 @@ @c %**end of header @copying -Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. -Copyright @copyright{} 2006-2007 Robert L. Read and Ian Eslick +Original Version, Copyright @copyright{} 2004 Ben Lee and Andrew Blumberg. +2006 SQL Backend, Copyright @copyright{} 2006 Robert L. Read. +2007 Rewrite, Copyright @copyright{} 2007 Ian Eslick. @quotation Permission is granted to copy, distribute and/or modify this document @@ -52,7 +53,7 @@ * User API Reference:: Function and class documentation of the user API. * Elephant Design:: An overview of elephant's internal architecture. * Data Store API Reference:: Function level documentation for data store implementors. -* Copying:: Your rights and freedoms. +* Copyright and License:: Your rights and freedoms. @end menu @chapheading Appendices @@ -66,6 +67,7 @@ @end menu @node Table of Contents +@unnumbered @comment node-name, next, previous, up @contents --- /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/03/24 12:16:02 1.4 +++ /project/elephant/cvsroot/elephant/doc/installation.texinfo 2007/03/30 14:34:34 1.5 @@ -6,157 +6,232 @@ @cindex Installation @menu -* Installation Basics:: Basic installation -* Test-Suites:: Running the test suites -* Berkeley DB Introduction:: The Berkeley DB backend -* SQL Data Store:: The design and status of the SQL back-end extension. -* Lisp Data Store:: A native lisp-based repository. -* Multi-repository Operation:: Specifying repositories. -* Setting up PostGres:: An example. +* Requirements:: Supported lisps and required libraries. +* Configuring Elephant:: Setting up Elephant and the configuration file. +* Loading Elephant:: Loading Elephant and the data store loading protocol. +* Berkeley DB Data Store:: Installing support for the Berkeley DB data store +* Berkeley DB Examples:: An example of installing and running the Berkeley DB data store. +* CL-SQL Data Store:: Install and connecting to the CL-SQL data store +* CL-SQL Examples:: An example of using the CL-SQL data store. +* Elephant on Windows:: More details about running Elephant on Windows +* Test Suites:: How to run and interpret the output of the regression test suite +* Documentation:: Building documentation from texinfo sources. @end menu -@node Installation Basics +@node Requirements @comment node-name, next, previous, up -@section Installation +@section Requirements -Please see the file ``INSTALL'' in the source distribution for -more precise information; this is an overview. +Elephant is a multi-platform, multi-lisp and multi-backend system. As +such there is a great deal of complexity in testing. The system has +tried to minimize external dependencies as much as possible to ease +installation, but it still requires some patience and care to bring +Elephant up on any given platform. This section attempts to simplify +this for new users as much as possible. Patches and suggestions will +be gladly accepted. + +@subsection Supported Lisp, Platform and Data store combinations + +Elephant supports SBCL, Allegro, Lispworks, OpenMCL and CMUCL. Each +lisp is supported on each of the platforms it runs on: Mac OS X, Linux +and Windows. As of release 0.6.1, both 32-bit and 64-bit systems +should be supported. Elephant has a small developer base and as of +the writing of this manual, there are: -Installation of Elephant itself is easy because of the asdf system. -Just execute: -@lisp -(asdf:operate 'asdf:load-op :elephant) -@end lisp +@enumerate +@item Five lisp environments +@item Three Operating System platforms +@item 32-bit or 64-bit OS/compilation configuration +@item Three data store configurations: Berkeley DB, SQLite3 and Postgresql +@end enumerate -However, Elephant cannot function without a back-end repository. -Elephant presents exactly the same API no matter what you choose -as a repository. In most cases Elephant will automatically load -the backend you refer to with your controller spec when you call -open store. However, you may have to use asdf to load the -code that interfaces to particular repository system. +This means that the total number of combinations that should be tested +comes to: -The basic choices are to use the BerkeleyDB system or -a SQL based system. You must perform one of these: -@lisp -(asdf:operate 'asdf:load-op :ele-clsql) -(asdf:operate 'asdf:load-op :ele-bdb) -@end lisp +@math{lisps * os * radix * dstore = 5 * 3 * 2 * 3 = 90 configurations} -If you choose a SQL based system, you may have to -load a specific package for that system, such as: +Of course not all of these combinations are valid, but the +implications of these combinatorics is that not every combination will +be tested in any given release. The developers and active user base +currently cover all three data store configurations on the following +platforms: + +@itemize +@item 32/64-bit SBCL on Linux and Mac OS X +@item 32-bit Lispworks on Windows and Mac OS X +@item 32-bit Allegro on Mac OS X +@end itemize + +The developers will do their best to accomodate users who are keen to +test other combinations, but practically these configurations will be +the most stable and reliable. Elephant is becoming quite stable in +general, so don't be afriad to try an unemphasized combination - +chances are it is just a little more work to bring it up. -@lisp -(asdf:operate 'asdf:load-op :ele-sqlite3) -@end lisp -or, for Postgres, -@lisp -(asdf:oos 'asdf:load-op :clsql-postgresql-socket) -@end lisp +@subsection Library dependencies -You will have to have the CL-SQL package installed. Following the -documentation for CL-SQL under the section ``How CLSQL finds and loads foreign -libraries'' you may need to do something like: -@lisp -(clsql:push-library-path "/usr/lib/") -@end lisp +The Elephant core system requires: -before doing -@lisp -(asdf:oos 'asdf:load-op :clsql-postgresql-socket) -@end lisp +@enumerate +@item asdf -- @uref{http://www.cliki.net/asdf} +@item uffi -- version 1.5.17 or later, @uref{http://uffi.b9.com/} or @uref{http://www.cliki.net/UFFI} +@item cl-base64 -- @uref{http://www.cliki.net/cl-base64} +@item gcc -- Your system needs GCC (or Cygwin) to build the Elephant C-based serializer library. (Precompiled DLL's are available for Windows platforms on the @uref{http://www.common-lisp.net/project/elephant/downloads.html, download page}. +@item rt -- The RT regression test sytem is required to run the test suite: @uref{http://www.cliki.net/RT} +@end enumerate + +Follow the instructions at these URLs to download and setup the +libraries. (Note: uffi and cl-base64 are +@uref{http://www.cliki.net/ASDF-Install, asdf-installable} for those +of you with asdf-install on your system) + +In addition to these libraries, each data store has their own +dependencies as discussed in @ref{Berkeley DB Data Store} and +@ref{CL-SQL Data Store}. + +@node Configuring Elephant +@comment node-name, next, previous, up +@section Configuring Elephant -in order for clsql to find the PostGres library libpq.so, for example. +Before you can load the elephant packages into your running lisp, you +need to setup the configuration file. First, copy the reference file +config.sexp from the root directory to my-config.sexp. my-config.sexp +contains a lisp reader-formatted list of key-value pairs that tells +elephant where to find various libraries, how to build libraries, etc. + +For example: -Without modifcation, Elephant uses this as it's lib path: @lisp -/usr/local/share/common-lisp/elephant-0.3/ -@end lisp +#+(and (or sbcl allegro) macosx) +((:berkeley-db-include-dir . "/opt/local/include/db45/") + (:berkeley-db-lib-dir . "/opt/local/lib/db45/") + (:berkeley-db-lib . "/opt/local/lib/db45/libdb-4.5.dylib") + (:berkeley-db-deadlock . "/opt/local/bin/db45_deadlock") + (:pthread-lib . nil) + (:clsql-lib . nil) + (:compiler . :gcc)) +@end lisp + +The following is a guide to the various parameters. For simplicity, +we include all the parameters here, although we will go into more +detail in each of the data store sections. + +@itemize +@item @strong{:compiler} -- This tells Elephant which compiler to use to build any C libraries. The only options currently are :gcc on Unix platforms and :cygwin for the Windows platform. +@item @strong{:berkeley-db-include-dir} -- The pathname for the Berkeley DB include files (db.h) +@item @strong{:berkeley-db-lib-dir} -- The pathname for all the Berkeley DB library files +@item @strong{:berkeley-db-lib} -- The full pathname for the specific Berkeley DB library (libdb45.so) +@item @strong{:berkeley-db-deadlock} -- The full pathname to the BDB utility function db_deadlock +@item @strong{:pthread-lib} -- Not needed for SBCL 9.17+ +@item @strong{:clsql-lib} -- Currently unused, adds paths to the CL-SQL library search function +@end itemize + +The config.sexp file contains a set of example configurations to start +from, but you will most likely need to modify it for your system. + +Elephant has one small C library that it uses for binary serialization +which means that you need to have gcc in your path (@pxref{Elephant on +Windows} for exceptions on the Windows platform). + +@node Loading Elephant +@comment node-name, next, previous, up +@section Loading Elephant -So you could put a symbolic link to libpq.so there, where libmemutil.so and -libsleepycat.so will also reside. +@subsection Loading Elephant via ASDF -Elephant is designed to allow multi-repository operation; -so you could concievably use two or more repositories at the -same time. More particularly, you can seamlessly migrate your -data from one repository to a different one at a later date. -In a long duration project, this might occur because of a licensing -or performance issue with a particular respository. Migrating to -a new repository of the same type is a cheap form of GC although -migration is limited to the total size of main memory to store -a hash table that tracks all copied object ID's. - -@node Test-Suites -@comment node-name, next, previous, up -@section Test-Suites - -Elephant is moderately mature. Hopefully, it will work out-of-the-box -for you. - -However, if you are using an LISP implementation different than the ones -on which it is developed and maintained (currently OpenMCL, SBCL, and ACL), -or as the repositories evolve, or just because of mistakes, you may need -to run the test suites. If you report a bug, we will ask you -to run these tests and report the output. Running them when you -first install things may give you a sense of confidence and understanding -that makes it worth the trouble. +Now that you have loaded all the dependencies and created your +configuration file you can load the Elephant packages and +definitions: -There are three files that execute the tests. You should choose -one as a starting point based on what backend(s) you are using. -If using BerekleyDB, use @lisp -BerkeleyDB-tests.lisp +(asdf:operate 'asdf:load-op :elephant) @end lisp -If using both, use both of the above and also use: +This will load the cl-base64 and uffi libraries. It will also +automatically compile and load the C library. The build process no +longer depends on a Makefile. This build process has been verified on +most platforms, but if you have a problem please report it, and any +output you can capture, to the developers at +@email{elephant-devel@@common-lisp.net}. We will update the FAQ at +@uref{http://trac.common-lisp.net/elephant} with common problems users +run into. + +@subsection Two-Phase Load Process + +Elephant uses a two-phase load process. The core code is loaded and +the code for a given data store is loaded on demand when you call +@code{open-store} with a specification referencing that data store. +The second phase of the load process requires ASDF to be installed on +your system. + +(NOTE: There are some good reasons and not so good reasons for this +process. One reason you cannot load ele-bdb.asd directly as it +depends on lisp code defined in elephant.asd. We decided not to fix +this in this release although later releases may avoid the oddity of +the two phase loading) + +@subsection Packages + +Now that Elephant has been loaded, you can call @code{use-package} in +the cl-user package or create a new package that imports the symbols +exported from package :elephant. + @lisp -MigrationTests.lisp +CL-USER> (use-package :elephant) +=> T + +OR + +(defpackage :elephant-user + (:use :common-lisp :elephant)) @end lisp -The text of this file is included here to give the -casual reader an idea of how elepant test can be run in general: -@lisp -;; This file is an example of how to perform the -;; migration tests. You will have to modify it -;; slightly depending on the systems that want to test... -;; You can test migration even between two BDB respositories if you wish -(asdf:operate 'asdf:load-op :elephant) -(asdf:operate 'asdf:load-op :ele-clsql) -(asdf:operate 'asdf:load-op :clsql-postgresql-socket) -(asdf:operate 'asdf:load-op :ele-bdb) -(asdf:operate 'asdf:load-op :elephant-tests) -;; For sqlite-3.. -;; (asdf:operate 'asdf:load-op :ele-sqlite3) +Beginners can skip to the end of this section. -(in-package "ELEPHANT-TESTS") +Elephant has a common package called elephant that exports a set of +generic functions. It also contains a dispatcher based on the first +element of a specification list that calls the relevant backend +version of @code{open-controller}, the internal method that creates a +@code{store-controller}. Each backend has it's own subclass +implementing the abstract interface of @code{store-controller}. -;; The primary and secondary test-paths are -;; use for the migration tests. +@subsection Opening a Store -;; This this configuration for testing between BDB and SQL.... -(setq *test-path-primary* *testpg-path*) -;; (setq *test-path-primary* *testsqlite3-path*) -(setq *test-path-secondary* *testdb-path*) +As discussed in the tutoral, you can now open a store to begin using +Elephant: -;; This this configuration for testing from one BDB repository to another... -(setq *test-path-primary* *testdb-path*) -;; (setq *test-path-primary* *testsqlite3-path*) -(setq *test-path-secondary* *testdb-path2*) +@lisp +(open-store '(:BDB "/Users/owner/db/my-bdb/")) +... +ASDF loading messages +... +=> #<BDB-STORE-CONTROLLER> -(do-migrate-test-spec *test-path-primary*) +(open-store '(:CLSQL (:POSTGRESQL "localhost.localdomain" "mydb" "myuser" "")))) +... +ASDF loading messages +... +=> #<SQL-STORE-CONTROLLER> @end lisp -The appropriate test should execute for you with no errors. -If you get errors, you may wish to report it the -@code{ elephant-devel at common-lisp.net} email list. +The first time you load a specific data store, Elephant will call ASDF +to load all the specified data store's dependencies, connect to a +database and return the @code{store-controller} subclass instance for +that backend. -@node Berkeley DB Repository +@node Berkeley DB Data Store @comment node-name, next, previous, up -@section Berkeley DB Repository +@section Berkeley DB Data Store + -@node SQL Repository +@node Berkeley DB Example +@comment node-name, next, previous, up +@section Setting up Berkeley DB + +@node CL-SQL Data Store @comment node-name, next, previous, up -@section SQL Repository +@section CL-SQL Data Store Although originally designed as an interface to the BerkeleyDB system, the original Elephant system has been experimenetally extended to @@ -227,44 +302,14 @@ the multi-repository version somewhat complicates the underlying persistent object management. -@node Multi-repository Operation +@node PostGres Examples @comment node-name, next, previous, up -@section Multi-repository Operation - -Elephant now keeps a small hashtables that maps ``database specifications'' into -actual database connections. - -If a database spec is a string, it is assumed to be a BerkeleyDB path. -If it is a list, it is a assumed to be a CL-SQL connection specification. -For example: -@lisp -ELE-TESTS> *testdb-path* -"/home/read/projects/elephant/elephant/tests/testdb/" -ELE-TESTS> *testpg-path* -(:postgresql "localhost.localdomain" "test" "postgres" "") -ELE-TESTS> -@end lisp - -The tests now have a function @code{do-all-tests-spec} that take a spec and -based on its type attempt to open the correct kind of store controller and -perform the tests. - -The routine @code{get-controller} takes this specifiation. - -The basic strategy is that the ``database specification'' object is stored in -every persistent object and collection so that the repository can be found. - -In this way, objects that reside in different repositories can coexist within -the LISP object space, allowing data migration. - - - +@section Setting up PostGres @node Setting up PostGres @comment node-name, next, previous, up @section Setting up PostGres - To set up a PostGres based back end, you should: @enumerate @@ -294,7 +339,6 @@ Before you attempt to connect with Elephant. [114 lines skipped] --- /project/elephant/cvsroot/elephant/doc/make-ref.lisp 2007/03/24 13:55:15 1.4 +++ /project/elephant/cvsroot/elephant/doc/make-ref.lisp 2007/03/30 14:34:34 1.5 @@ -16,11 +16,14 @@ (load docstrings-path) (defun make-docs () - (when t - (elephant:open-store elephant-tests::*testbdb-spec*) - (make-instance 'elephant::persistent-collection) - (make-instance 'elephant::secondary-cursor) - (make-instance 'elephant::indexed-btree) - (sb-texinfo:generate-includes #p"/Users/eslick/Work/fsrc/elephant-cvs/doc/includes/" (find-package :elephant) (find-package :elephant-backend) (find-package 'elephant-memutil) (find-package 'elephant-system)))) + (elephant:open-store elephant-tests::*testbdb-spec*) + (make-instance 'elephant::persistent-collection) + (make-instance 'elephant::secondary-cursor) + (make-instance 'elephant::indexed-btree) + (sb-texinfo:generate-includes #p"/Users/eslick/Work/fsrc/elephant-cvs/doc/includes/" + (find-package :elephant) + (find-package :elephant-backend) + (find-package :elephant-memutil) + (find-package :elephant-system))) (make-docs) --- /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/03/24 12:16:02 1.6 +++ /project/elephant/cvsroot/elephant/doc/reference.texinfo 2007/03/30 14:34:34 1.7 @@ -14,7 +14,6 @@ * Collections:: BTrees and indices. * Cursors:: Traversing BTrees. * Transactions:: Transactions. -* Multithreading:: Multithreading. * Migration and Upgrading:: Migration and upgrading. @end menu @@ -56,23 +55,48 @@ @section Persistent Object Indexing @cindex Persistent Object Indexing -@include includes/fun-get-instances-by-class.texinfo -@include includes/fun-get-instance-by-value.texinfo -@include includes/fun-get-instances-by-value.texinfo -@include includes/fun-get-instances-by-range.texinfo +@subsection Indexed Object Accessors + +@include includes/fun-elephant-map-class.texinfo +@include includes/fun-elephant-map-class-index.texinfo + +@include includes/fun-elephant-get-instances-by-class.texinfo +@include includes/fun-elephant-get-instance-by-value.texinfo +@include includes/fun-elephant-get-instances-by-value.texinfo +@include includes/fun-elephant-get-instances-by-range.texinfo + +@include includes/fun-elephant-drop-instances.texinfo + +@subsection Direct Class Index Manipulation + +@include includes/fun-elephant-find-class-index.texinfo +@include includes/fun-elephant-find-inverted-index.texinfo +@include includes/fun-elephant-make-class-cursor.texinfo +@include includes/macro-elephant-with-class-cursor.texinfo +@include includes/fun-elephant-make-inverted-cursor.texinfo +@include includes/macro-elephant-with-inverted-cursor.texinfo + +@subsection Dynamic Indexing API @include includes/fun-elephant-enable-class-indexing.texinfo @include includes/fun-elephant-disable-class-indexing.texinfo -@include includes/fun-add-class-slot-index.texinfo -@include includes/fun-remove-class-slot-index.texinfo -@include includes/fun-add-class-derived-index.texinfo -@include includes/fun-remove-class-derived-index.texinfo +@include includes/fun-elephant-add-class-slot-index.texinfo +@include includes/fun-elephant-remove-class-slot-index.texinfo +@include includes/fun-elephant-add-class-derived-index.texinfo +@include includes/fun-elephant-remove-class-derived-index.texinfo @node Query Interfaces @comment node-name, next, previous, up @section Query Interfaces @cindex Query Interfaces +Query interfaces are currently unimplemented. An example query +interface is provided for reference only, a new system is under +development for the 0.7 release. + +@include includes/fun-elephant-get-query-results.texinfo +@include includes/fun-elephant-map-class-query.texinfo + @node Collections @comment node-name, next, previous, up @section Collections @@ -145,12 +169,9 @@ @include includes/fun-elephant-commit-transaction.texinfo @include includes/fun-elephant-abort-transaction.texinfo -@node Multithreading -@comment node-name, next, previous, up -@section Multithreading -@cindex Multithreading - @node Migration and Upgrading @comment node-name, next, previous, up @section Migration and Upgrading @cindex Migration and Upgrading + +@include includes/fun-elephant-migrate.texinfo --- /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/03/26 03:37:27 1.9 +++ /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/03/30 14:34:34 1.10 @@ -914,8 +914,8 @@ complicated. The best strategy at the beginning is a conservative one, break things up into the smallest logical sets of primitive operations and only wrap higher level functions in transactions when -they absolutely have to commit together. @xref{Transaction details} -for all the gory detail of transactions and @pxref{Usage scenarios} -for more examples of how systems can be designed using transactions. +they absolutely have to commit together. See @ref{Transaction details} +for the full details and @pxref{Usage scenarios} for more examples of +how systems can be designed and tuned using transactions. --- /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/03/26 03:37:27 1.3 +++ /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/03/30 14:34:34 1.4 @@ -16,6 +16,7 @@ * Secondary Indices:: Alternative ways to index collections. * Using Cursors:: Low-level access to BTrees. * Transaction details:: Develop a deeper understanding of transactions and avoid the pitfalls. +* Multi-repository Operation:: Specifying repositories. * Repository Migration and Upgrade:: How to move objects from one repository to another. * Garbage collection:: How to recover storage and OIDs in long-lived repositories. * Performance tuning:: How to get the most from Elephant. @@ -82,10 +83,9 @@ @code{with-open-controller} macro. Opening and closing a controller is very expensive. - -@node{Class indices} +@node Class Indices @comment node-name, next, previous, up -@section Class indicies +@section Class Indicies You can enable/disable class indexing for an entire class. When you disable indexing all references to instances of that class are lost. If you re-enable @@ -130,7 +130,7 @@ somewhat user customizable; documentation for this exists in the source file referenced above. -@node{Using BTrees} +@node Using BTrees @comment node-name, next, previous, up @section Using BTrees @@ -225,7 +225,7 @@ to the target repository which you can then overwrite. To avoid the default persistent slot copying, bind the dynamic variable @code{*inhibit-slot-writes*} in your user method using -@code(with-inhibited-slot-copy () ...)} a convenience macro. +@code{with-inhibited-slot-copy} a convenience macro. @node Threading @@ -318,3 +318,33 @@ ensure that multiple threads do not interleave access so single user mode is not suitable for use in web servers or other typically multi-threaded applications. + +@node Multi-repository Operation +@comment node-name, next, previous, up +@section Multi-repository Operation + +Elephant now keeps a small hashtables that maps ``database specifications'' into +actual database connections. + +If a database spec is a string, it is assumed to be a BerkeleyDB path. +If it is a list, it is a assumed to be a CL-SQL connection specification. +For example: +@lisp +ELE-TESTS> *testdb-path* +"/home/read/projects/elephant/elephant/tests/testdb/" +ELE-TESTS> *testpg-path* +(:postgresql "localhost.localdomain" "test" "postgres" "") +ELE-TESTS> +@end lisp + +The tests now have a function @code{do-all-tests-spec} that take a spec and +based on its type attempt to open the correct kind of store controller and +perform the tests. + +The routine @code{get-controller} takes this specifiation. + +The basic strategy is that the ``database specification'' object is stored in +every persistent object and collection so that the repository can be found. + +In this way, objects that reside in different repositories can coexist within +the LISP object space, allowing data migration.

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv31032/src/db-bdb Modified Files: bdb-controller.lisp Log Message: Significant documentation string and documentation edits towards 0.6.1 manual. Clean up packages so elephant exports user visible symbols and backend exports backend-relevant symbols. Change required fix in serializer packages also. Added :elephant-user package. --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/03/16 14:44:44 1.31 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/03/30 14:34:35 1.32 @@ -135,7 +135,7 @@ (db-open db :file "%ELEPHANTOID" :database "%ELEPHANTOID" :auto-commit t :type DB-BTREE :create t :thread thread) (let ((oid-seq (db-sequence-create db))) - (db-sequence-set-cachesize oid-seq *cachesize*) + (db-sequence-set-cachesize oid-seq 100) (db-sequence-set-flags oid-seq :seq-inc t :seq-wrap t) (db-sequence-set-range oid-seq 0 most-positive-fixnum) (db-sequence-initial-value oid-seq 0)

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv31032/src/elephant Modified Files: backend.lisp classindex.lisp controller.lisp package.lisp query.lisp serializer.lisp serializer1.lisp serializer2.lisp variables.lisp Log Message: Significant documentation string and documentation edits towards 0.6.1 manual. Clean up packages so elephant exports user visible symbols and backend exports backend-relevant symbols. Change required fix in serializer packages also. Added :elephant-user package. --- /project/elephant/cvsroot/elephant/src/elephant/backend.lisp 2007/02/16 23:02:53 1.11 +++ /project/elephant/cvsroot/elephant/src/elephant/backend.lisp 2007/03/30 14:34:35 1.12 @@ -23,108 +23,33 @@ (:documentation "Backends should use this to get access to internal symbols of elephant that importers of elephant shouldn't see. Backends should also import elephant to get use-api generic function symbols, classes and globals") + (:use #:elephant) (:import-from #:elephant ;; Variables - #:*cachesize* - #:*dbconnection-spec* ;; shouldn't need this + #:*dbconnection-spec* #:connection-is-indeed-open - ;; Persistent objects - #:oid #:get-con - #:next-oid - #:persistent-slot-writer - #:persistent-slot-reader - #:persistent-slot-boundp - #:persistent-slot-makunbound - ;; Controllers - #:*elephant-code-version* - #:store-controller - #:open-controller - #:database-version - #:close-controller - #:controller-serialize - #:controller-deserialize - #:controller-spec - #:controller-root - #:controller-version - #:controller-class-root - #:root #:spec #:class-root - #:flush-instance-cache - #:controller-symbol-cache #:controller-symbol-id-cache - ;; Collection generic functions - #:btree #:btree-index #:indexed-btree - #:build-indexed-btree #:build-btree #:existsp - #:map-indices - ;; Serialization - #:deserialize #:serialize - #:deserialize-from-base64-string - #:serialize-to-base64-string - ;; Serialization callbacks - #:lookup-persistent-symbol - #:lookup-persistent-symbol-id - ;; Cursor accessors - #:cursor - #:cursor-btree - #:cursor-oid - #:cursor-initialized-p - ;; Transactions - #:*current-transaction* - #:make-transaction-record - #:transaction-store - #:transaction-object - #:execute-transaction - #:controller-start-transaction - #:controller-commit-transaction - #:controller-abort-transaction - ;; Misc - #:slot-definition-name - #:remove-indexed-element-and-adjust - #:register-backend-con-init - #:lookup-backend-con-init - ;; Lisp specific - #+(or sbcl cmu) #:%bignum-ref - ) - (:export - ;; Variables - #:*cachesize* - #:*dbconnection-spec* ;; shouldn't need this - #:connection-is-indeed-open ;; Persistent objects #:oid #:get-con - #:next-oid + #:next-oid #:persistent-slot-writer #:persistent-slot-reader #:persistent-slot-boundp #:persistent-slot-makunbound + ;; Controllers #:*elephant-code-version* - #:store-controller #:open-controller - #:database-version #:close-controller + #:database-version + #:controller-spec #:controller-serialize #:controller-deserialize - #:controller-spec - #:controller-root - #:controller-class-root - #:controller-version #:root #:spec #:class-root - #:flush-instance-cache - #:controller-symbol-cache #:controller-symbol-id-cache - ;; Collection generic functions - #:btree #:btree-index #:indexed-btree - #:build-indexed-btree #:build-btree #:existsp - #:map-indices ;; Serialization - #:deserialize #:serialize - #:serialize-symbol-complete #:deserialize-from-base64-string #:serialize-to-base64-string - ;; Serialization callbacks - #:lookup-persistent-symbol - #:lookup-persistent-symbol-id ;; Cursor accessors - #:cursor #:cursor-btree #:cursor-oid #:cursor-initialized-p @@ -133,14 +58,58 @@ #:make-transaction-record #:transaction-store #:transaction-object - #:execute-transaction - #:controller-start-transaction - #:controller-commit-transaction - #:controller-abort-transaction + ;; Registration + #:register-backend-con-init + #:lookup-backend-con-init ;; Misc #:slot-definition-name + #:slots-and-values + #:struct-slots-and-values #:remove-indexed-element-and-adjust - #:register-backend-con-init - #:lookup-backend-con-init - )) + ) + (:export + ;; Variables + #:*cachesize* + #:*dbconnection-spec* + #:connection-is-indeed-open + + ;; Persistent objects + #:oid #:get-con + #:next-oid + #:persistent-slot-writer + #:persistent-slot-reader + #:persistent-slot-boundp + #:persistent-slot-makunbound + + ;; Controllers + #:*elephant-code-version* + #:open-controller + #:close-controller + #:database-version + #:controller-spec + #:controller-version + #:controller-serialize + #:controller-deserialize + #:root #:spec #:class-root + ;; Serialization + #:deserialize-from-base64-string + #:serialize-to-base64-string + ;; Cursor accessors + #:cursor-btree + #:cursor-oid + #:cursor-initialized-p + ;; Transactions + #:*current-transaction* + #:make-transaction-record + #:transaction-store + #:transaction-object + ;; Registration + #:register-backend-con-init + #:lookup-backend-con-init + ;; Misc + #:slot-definition-name + #:slots-and-values + #:struct-slots-and-values + #:remove-indexed-element-and-adjust + )) --- /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/03/25 14:57:49 1.33 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/03/30 14:34:35 1.34 @@ -141,7 +141,7 @@ (map-btree (lambda (class-name index) (declare (ignore index)) (let ((class (find-class class-name nil))) - (when class + (when (and class (subtypep class 'persistent-metaclass)) (setf (%index-cache class) nil)))) (controller-class-root sc))) (t (e) (warn "Unable to clear class index caches ~A" e))))) @@ -333,16 +333,22 @@ ;; USER CURSOR API ;; =================== -(defgeneric make-inverted-cursor (persistent-metaclass name) +(defgeneric make-inverted-cursor (class name) (:documentation "Define a cursor on the inverted (slot or derived) index")) -(defgeneric make-class-cursor (persistent-metaclass) +(defgeneric make-class-cursor (class) (:documentation "Define a cursor over all class instances")) + (defmethod make-inverted-cursor ((class persistent-metaclass) name) (make-cursor (find-inverted-index class name))) +(defmethod make-inverted-cursor ((class symbol) name) + (make-cursor (find-inverted-index class name))) + (defmacro with-inverted-cursor ((var class name) &body body) + "Bind the var argument to an inverted cursor on the index + specified the provided class and index name" `(let ((,var (make-inverted-cursor ,class ,name))) (unwind-protect (progn ,@body) (cursor-close ,var)))) @@ -350,7 +356,12 @@ (defmethod make-class-cursor ((class persistent-metaclass)) (make-cursor (find-class-index class))) +(defmethod make-class-cursor ((class symbol)) + (make-cursor (find-class-index class))) + (defmacro with-class-cursor ((var class) &body body) + "Bind the var argument in the body to a class cursor on the + index specified the provided class or class name" `(let ((,var (make-class-cursor ,class))) (unwind-protect (progn ,@body) (cursor-close ,var)))) @@ -361,8 +372,8 @@ ;; ====================== (defun map-class (fn class) - "Perform a map operation across all instances of class. Takes a - function of one argument, the class instance" + "Perform a map operation over all instances of class. Takes a + function of one argument, a class instance" (let* ((class (if (symbolp class) (find-class class) class)) @@ -374,9 +385,22 @@ (map-btree #'map-fn class-idx)))) (defun map-class-index (fn class index &rest args &key start end value) - "To map over a subset of instances, pick an index by slot name - or derived index name and specify the bounds for the traversal. - Otherwise use map-class for all instances. " + "This function maps over a subset of class instances in the + order defined by the index. Specify the class and index by + quoted name. The index may be a slot index or a derived + index. + + To map only a subset of key-value pairs, specify the range + using the :start and :end keywords; all elements greater than + or equal to :start and less than or equal to :end will be + traversed regardless of whether the start or end value is in + the index. + + Use nil in the place of start or end to specify the first + element or last element, respectively. + + To map a single value, iff it exists, use the :value keyword. + This is the only way to travers all nil values." (declare (dynamic-extent args) (ignorable args)) (let* ((index (if (symbolp index) @@ -395,11 +419,14 @@ (defgeneric get-instances-by-class (persistent-metaclass) (:documentation "Retrieve all instances from the class index as a list of objects")) + (defgeneric get-instance-by-value (persistent-metaclass slot-name value) (:documentation "Retrieve instances from a slot index by value. Will return only the first instance if there are duplicates.")) + (defgeneric get-instances-by-value (persistent-metaclass slot-name value) (:documentation "Returns a list of all instances where the slot value is equal to value.")) + (defgeneric get-instances-by-range (persistent-metaclass slot-name start end) (:documentation "Returns a list of all instances that match values between start and end. An argument of @@ -458,6 +485,8 @@ (nreverse instances))) (defun drop-instances (instances &key (sc *store-controller*)) + "Removes a list of persistent objects from all class indices + and unbinds any slot values" (when instances (assert (consp instances)) (do-subsets (subset 500 instances) --- /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/03/24 03:03:00 1.41 +++ /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/03/30 14:34:35 1.42 @@ -23,16 +23,23 @@ ;; TRACKING OBJECT STORES ;; -(defparameter *elephant-backends* +(defvar *elephant-backends* '((:bdb (:ele-bdb)) (:clsql (:ele-clsql)) ) - "Entries have the form of (backend-type asdf-depends-list") + "Tells the main elephant code the tag used in a store spec to + refer to a given backend. The second argument is an asdf + dependency list. Entries have the form of (backend-type + asdf-depends-list") (defvar *elephant-controller-init* (make-hash-table)) (defun register-backend-con-init (name controller-init-fn) - "Backends call this during evalution to register their init function's name" + "Backends must call this function during the + loading/compilation process to register their initialization + function for the tag name in *elephant-backends*. The + initialization function returns a fresh instance of the + backends store-controller subclass" (setf (gethash name *elephant-controller-init*) controller-init-fn)) (defun lookup-backend-con-init (name) @@ -102,6 +109,9 @@ ;; (defun get-user-configuration-parameter (name) + "This function pulls a value from the key-value pairs stored in + my-config.sexp so backends can have their own pairs for appropriate + customization after loading." (elephant-system::get-config-option name (asdf:find-system :elephant))) @@ -114,26 +124,51 @@ ((spec :type list :accessor controller-spec :initarg :spec - :documentation "Backend create functions should pass in :spec during make-instance") + :documentation "Backend initialization functions are + expected to initialize :spec on the call to + make-instance") ;; Generic support for the object, indexing and root protocols (root :reader controller-root - :documentation "This should be a persistent btree instantiated by the backend") + :documentation "This is an instance of the backend + persistent btree. It should have an OID that is fixed in + the code and does not change between sessions. Usually + it this is something like 0, 1 or -1") (class-root :reader controller-class-root - :documentation "This should be a persistent indexed btree instantiated by the backend") + :documentation + "This is another root for class indexing that is + also a backend specific persistent btree instance + with a unique OID that persists between sessions.") (instance-cache :accessor instance-cache :initform (make-cache-table :test 'eql) - :documentation "This is an instance cache and part of the metaclass - protocol. Backends should not override") + :documentation + "This is an instance cache and part of the + metaclass protocol. Backends should not + override the default behavior.") (instance-cache-lock :accessor instance-cache-lock :initform (ele-make-lock) - :documentation "Protection for updates to the cache from multiple threads") + :documentation "Protection for updates to + the cache from multiple threads. Do not + override.") ;; Upgradable serializer strategy - (serializer-version :accessor controller-serializer-version :initform nil) - (serialize :accessor controller-serialize :initform nil) - (deserialize :accessor controller-deserialize :initform nil) - ) + (serializer-version :accessor controller-serializer-version :initform nil + :documentation "Governs the default + behavior regarding which serializer + version the current elephant core is + using. Backends can override by creating + a method on initialize-serializer.") + (serialize :accessor controller-serialize :initform nil + :documentation "Accessed by elephant::serialize to + get the entry point to the default serializer or to + a backend-specific serializer") + (deserialize :accessor controller-deserialize :initform nil + :documentation "Contains the entry point for the + specific serializer to be called by + elephant::deserialize")) (:documentation - "Class of objects responsible for the book-keeping of holding DB - handles, the cache, table creation, counters, locks, the root - (for garbage collection,) et cetera.")) + "Superclass for the data store controller, the main interface + to any book-keeping, references to DB handles, the instance + cache, btree table creation, counters, locks, the roots (for + garbage collection,) et cetera. Behavior is shared between + the superclass and subclasses. See slot documentation for + details.")) ;; ;; Per-controller instance caching @@ -324,12 +359,22 @@ (defgeneric open-controller (sc &key recover recover-fatal thread &allow-other-keys) (:documentation "Opens the underlying environment and all the necessary -database tables.")) +database tables. Different backends may use different keys so +all methods should &allow-other-keys. There are three standard +keywords: :recover, :recover-fatal and :thread. Recover means +that recovery should be checked for or performed on startup. +Recover fatal means a full rebuild from log files is requested. +Thread merely indicates to the backend that it is a threaded +application and any steps that need to be taken (for example +transaction implementation) are taken. :thread is usually +true.")) (defgeneric close-controller (sc) (:documentation - "Close the db handles and environment. Tries to wipe out -references to the db handles.")) + "Close the db handles and environment. Should be in a state + where lisp could be shut down without causing an inconsistent + state in the db. Also, the object could be used by + open-controller to reopen the database")) (defmethod close-controller :after ((sc store-controller)) "Delete connection spec so store-controller operations on cached @@ -422,42 +467,53 @@ (defun add-to-root (key value &key (sc *store-controller*)) "Add an arbitrary persistent thing to the root, so you can -retrieve it in a later session. N.B. this means it (and -everything it points to) won't get gc'd." + retrieve it in a later session. Anything referenced by an + object added to the root is considered reachable and thus live" (declare (type store-controller store-controller)) (assert (not (eq key *elephant-properties-label*))) (setf (get-value key (controller-root sc)) value)) (defun get-from-root (key &key (sc *store-controller*)) - "Get a something from the root." + "Get the value associated with key from the root. Returns two + values, the value, or nil, and a boolean indicating whether a + value was found or not (so you know if nil is a value or an + indication of non-presence)" (declare (type store-controller sc)) (get-value key (controller-root sc))) (defun root-existsp (key &key (sc *store-controller*)) - "Test whether a key exists in the root" + "Test whether a given key is instantiated in the root" (declare (type store-controller sc)) (if (existsp key (controller-root sc)) t nil)) (defun remove-from-root (key &key (sc *store-controller*)) - "Remove something from the root." + "Remove something from the root by the key value" (declare (type store-controller sc)) (remove-kv key (controller-root sc))) (defun map-root (fn &key (sc *store-controller*)) - "Map over all key-value pairs in the root" + "Takes a function of two arguments, key and value, to map over + all key-value pairs in the root" (map-btree fn (controller-root sc))) ;; ;; Explicit storage reclamation ;; +(defgeneric drop-pobject (persistent-object) + (:documentation "drop-pobject reclaims persistent object storage by unbinding + all persistent slot values. It can also helps catch errors + where an object should be unreachable, but a reference still + exists elsewhere in the DB. On access, the unbound slots + should flag an error in the application program. IMPORTANT: + this function does not clear the cached object instance or any + serialized references still in the db. Need a migration or GC + for that! drop-instances is preferred as it implements the proper + behavior for indexed classes")) + (defmethod drop-pobject ((inst persistent-object)) - "Reclaim persistent object storage by unbinding slot values. - This does not delete the cached object instance or any - serialized references still in the db. - Need a migration or GC for that!" (let ((pslots (persistent-slots (class-of inst)))) (dolist (slot pslots) (slot-makunbound inst slot)))) --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/03/23 16:08:10 1.25 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/03/30 14:34:35 1.26 @@ -25,107 +25,6 @@ (:documentation "Elephant: an object-oriented database for Common Lisp with multiple backends for Berkeley DB, SQL and others.") - (:export #:*store-controller* #:*current-transaction* - #:*elephant-lib-path* #:*elephant-code-version* - #:with-elephant-variables - - #:store-controller #:controller-root #:controller-class-root - #:open-store #:close-store #:with-open-store - #:add-to-root #:get-from-root #:remove-from-root #:root-existsp - #:map-root #:get-cached-instance #:flush-instance-cache - #:controller-symbol-cache #:controller-symbol-id-cache - #:controller-fast-symbols-p - #:optimize-layout #:drop-pobject - #:get-user-configuration-parameter - #:database-version - - #:upgrade - - #:controller-version #:controller-serializer-version - #:controller-serialize #:controller-deserialize - #:serialize-database-version-key - #:serialize-database-version-value - #:deserialize-database-version-value - #:serialize-database-serializer-version-value - #:deserialize-database-serializer-version-value - #:initialize-serializer - - #:with-transaction #:ensure-transaction - #:start-ele-transaction #:commit-transaction #:abort-transaction - - #:persistent #:persistent-object #:persistent-metaclass - #:persistent-collection #:defpclass - - #:btree #:make-btree #:get-value #:remove-kv #:existp - #:indexed-btree #:make-indexed-btree - #:add-index #:get-index #:remove-index #:map-indices - #:btree-index #:get-primary-key - #:primary #:key-form #:key-fn - - #:struct-constructor - - #:migrate #:set-oid-spec #:*inhibit-slot-copy* - #:add-symbol-conversion #:add-package-conversion - #:*always-convert* - - #:translate-and-intern-symbol - #:lookup-persistent-symbol - #:lookup-persistent-symbol-id - #:int-byte-spec - - #:cursor #:secondary-cursor #:make-cursor #:make-simple-cursor - #:cursor-close #:cursor-init - #:cursor-duplicate #:cursor-current #:cursor-first - #:cursor-last #:cursor-next #:cursor-next-dup - #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup - #:cursor-set #:cursor-set-range #:cursor-get-both - #:cursor-get-both-range #:cursor-delete #:cursor-put - #:cursor-pcurrent #:cursor-pfirst #:cursor-plast - #:cursor-pnext #:cursor-pnext-dup #:cursor-pnext-nodup - #:cursor-pprev #:cursor-pprev-nodup #:cursor-pset - #:cursor-pset-range #:cursor-pget-both - #:cursor-pget-both-range - - ;; Class indexing management API - #:*default-indexed-class-synch-policy* - #:find-class-index #:find-inverted-index - #:enable-class-indexing #:disable-class-indexing - #:add-class-slot-index #:remove-class-slot-index - #:add-class-derived-index #:remove-class-derived-index - #:describe-db-class-index - #:report-indexed-classes - #:class-indexedp-by-name - - ;; Low level cursor API - #:make-inverted-cursor #:make-class-cursor - #:with-inverted-cursor #:with-class-cursor - - ;; Primitive mapping API - #:with-btree-cursor - #:map-btree - #:map-index - - ;; BTREE Utilities - #:empty-btree-p - #:dump-btree - #:btree-keys - #:btree-differ-p - - ;; Class mapping API - #:map-class - #:map-class-index - - ;; Instance query API - #:get-instances-by-class - #:get-instance-by-value - #:get-instances-by-value - #:get-instances-by-range - #:drop-instances - - ;; Utilities - #:slots-and-values - #:struct-slots-and-values - ) #+cmu (:import-from :pcl compute-class-precedence-list @@ -296,7 +195,71 @@ slot-definition-allocation slot-definition-initargs compute-slots) + (:export + #:*store-controller* + #:store-controller #:controller-root #:controller-class-root + #:open-store #:close-store #:with-open-store + #:add-to-root #:get-from-root #:remove-from-root #:root-existsp #:map-root + #:flush-instance-cache + #:optimize-layout + + #:persistent #:persistent-object #:persistent-metaclass #:defpclass + #:persistent-collection #:drop-pobject + + #:btree #:make-btree + #:get-value #:remove-kv #:existp + #:indexed-btree #:make-indexed-btree + #:add-index #:get-index #:remove-index #:map-indices + #:get-primary-key #:primary #:key-form #:key-fn + #:with-btree-cursor #:map-btree #:map-index + #:empty-btree-p #:dump-btree #:btree-keys #:btree-differ-p + + #:cursor #:secondary-cursor #:make-cursor #:make-simple-cursor + #:cursor-close #:cursor-init + #:cursor-duplicate #:cursor-current #:cursor-first + #:cursor-last #:cursor-next #:cursor-next-dup + #:cursor-next-nodup #:cursor-prev #:cursor-prev-nodup + #:cursor-set #:cursor-set-range #:cursor-get-both + #:cursor-get-both-range #:cursor-delete #:cursor-put + #:cursor-pcurrent #:cursor-pfirst #:cursor-plast + #:cursor-pnext #:cursor-pnext-dup #:cursor-pnext-nodup + #:cursor-pprev #:cursor-pprev-nodup #:cursor-pset + #:cursor-pset-range #:cursor-pget-both + #:cursor-pget-both-range + + #:find-class-index #:find-inverted-index + #:enable-class-indexing #:disable-class-indexing + #:add-class-slot-index #:remove-class-slot-index + #:add-class-derived-index #:remove-class-derived-index + #:describe-db-class-index + #:report-indexed-classes + #:class-indexedp-by-name + + #:map-class #:map-class-index + #:get-instances-by-class + #:get-instance-by-value + #:get-instances-by-value + #:get-instances-by-range + #:drop-instances + #:make-inverted-cursor #:make-class-cursor + #:with-inverted-cursor #:with-class-cursor + #:*default-indexed-class-synch-policy* + + #:with-transaction #:ensure-transaction + #:controller-start-transaction + #:controller-abort-transaction + #:controller-commit-transaction + + #:upgrade #:migrate + #:set-oid-spec #:*inhibit-slot-copy* + #:add-symbol-conversion #:add-package-conversion + #:*always-convert* + #:translate-and-intern-symbol + #:lookup-persistent-symbol + #:lookup-persistent-symbol-id + #:struct-constructor + ) ) (in-package "ELE") @@ -304,3 +267,10 @@ #+cmu (eval-when (:compile-toplevel) (proclaim '(optimize (ext:inhibit-warnings 3)))) + +(defpackage :elephant-user + (:use :common-lisp :elephant) + (:nicknames :ele-user) + (:documentation + "A user package for experimenting with Elephant")) + \ No newline at end of file --- /project/elephant/cvsroot/elephant/src/elephant/query.lisp 2007/03/01 02:45:45 1.1 +++ /project/elephant/cvsroot/elephant/src/elephant/query.lisp 2007/03/30 14:34:35 1.2 @@ -40,6 +40,7 @@ (number (funcall (relation-number-function rel) ival (first tvals))))) (defun get-query-instances (constraints) + "Get a list of instances according to the query constraints" (let ((list nil)) (flet ((collect (inst) (push inst list))) @@ -48,7 +49,9 @@ (defun map-class-query (fn constraints) "Map instances using the query constaints to filter objects, exploiting - slot indices (for last query) and stack allocated test closures" + slot indices (for last query) and stack allocated test closures. This is + a minimally optimizing version that uses the first index it finds, and + then does a nested loop join on the rest of the parameters." (assert (not (null constraints))) (destructuring-bind (class slot relation &rest values) (first constraints) (flet ((filter-by-relation (inst) --- /project/elephant/cvsroot/elephant/src/elephant/serializer.lisp 2007/03/03 17:24:59 1.25 +++ /project/elephant/cvsroot/elephant/src/elephant/serializer.lisp 2007/03/30 14:34:35 1.26 @@ -92,18 +92,21 @@ ;; Database Version (a list of integers = [version major minor]) (defun serialize-database-version-key (bs) + "Given a buffer-stream, encode a key indicating the version using + the constant +elephant-version+" (serialize-reserved-tag bs) (serialize-system-tag +elephant-version+ bs)) (defun serialize-database-version-value (version bs) - "Simple serializes a list containing three integers" - (assert (consp version)) + "Serializes a list containing three integers to the buffer stream bs" + (assert (and (= (length version) 3))) (destructuring-bind (version major minor) version (serialize-system-integer version bs) (serialize-system-integer major bs) (serialize-system-integer minor bs))) (defun deserialize-database-version-value (bs) + "Deserializes the 3 integer list from buffer stream bs" (let ((version (deserialize-system-integer bs)) (major (deserialize-system-integer bs)) (minor (deserialize-system-integer bs))) --- /project/elephant/cvsroot/elephant/src/elephant/serializer1.lisp 2007/02/26 19:12:18 1.12 +++ /project/elephant/cvsroot/elephant/src/elephant/serializer1.lisp 2007/03/30 14:34:35 1.13 @@ -24,11 +24,12 @@ #+sbcl (:import-from :sb-bignum %bignum-ref) - (:import-from :elephant + (:import-from #:elephant get-cached-instance slot-definition-allocation slot-definition-name compute-slots + slots-and-values oid int-byte-spec array-type-from-byte --- /project/elephant/cvsroot/elephant/src/elephant/serializer2.lisp 2007/03/21 14:29:31 1.33 +++ /project/elephant/cvsroot/elephant/src/elephant/serializer2.lisp 2007/03/30 14:34:35 1.34 @@ -29,6 +29,8 @@ slot-definition-allocation slot-definition-name compute-slots + slots-and-values + struct-slots-and-values oid int-byte-spec array-type-from-byte --- /project/elephant/cvsroot/elephant/src/elephant/variables.lisp 2007/02/02 23:51:58 1.10 +++ /project/elephant/cvsroot/elephant/src/elephant/variables.lisp 2007/03/30 14:34:35 1.11 @@ -38,9 +38,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Optimization parameters -(defvar *cachesize* 100 - "Size of the OID sequence cache.") - (defvar *circularity-initial-hash-size* 50 "This is the default size of the circularity cache used in the serializer")

1 0

[elephant-cvs] CVS elephant
by ieslick 30 Mar '07

30 Mar '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv31032 Modified Files: TODO elephant.asd Log Message: Significant documentation string and documentation edits towards 0.6.1 manual. Clean up packages so elephant exports user visible symbols and backend exports backend-relevant symbols. Change required fix in serializer packages also. Added :elephant-user package. --- /project/elephant/cvsroot/elephant/TODO 2007/03/24 12:16:02 1.75 +++ /project/elephant/cvsroot/elephant/TODO 2007/03/30 14:34:34 1.76 @@ -18,7 +18,7 @@ - Verify db_deadlock for other lisps (launch and kill background program I/F) Bugs: -- Fix any bugs found during BETA +- Support for asdf-install? Test coverage: - Clean up interface to tests @@ -47,6 +47,20 @@ 0.6.1 - Features COMPLETED to date ---------------------------------- +POST BETA CHECKINS: + +Bugs: +x Fix duplicate opening of CLSQL db bug that caused errors in SQLite +x Fix for persistent-object inheritance calculation when inheriting from standard classes +x Remove problematic asserts in lisp-compare-eq make equal for strings instead of eq +x Fix map over nils bug in map-index and get-instances-by-value; clarify map-index interface + +Tweaks: +x Add bounds to map-btree +x Change wipe-class-indexing so it can be called without the class object being created +x add-to-root, etc now uses :sc instead of :store-controller for brevity +x Remove hard coded paths in ele-clsql + POST ALPHA CHECKINS: Major Bugs: --- /project/elephant/cvsroot/elephant/elephant.asd 2007/03/19 20:35:30 1.38 +++ /project/elephant/cvsroot/elephant/elephant.asd 2007/03/30 14:34:34 1.39 @@ -294,11 +294,11 @@ (:file "transactions") (:file "metaclasses") (:file "classes") + (:file "cache") (:file "serializer") (:file "serializer1") ;; 0.6.0 db's (:file "serializer2") ;; 0.6.1 db's (:file "unicode2") - (:file "cache") (:file "controller") (:file "collections") (:file "classindex-utils") @@ -309,4 +309,3 @@ :depends-on (memutil utils))))) :serial t :depends-on (:uffi :cl-base64)) -

1 0

[elephant-cvs] CVS elephant/doc
by ieslick 26 Mar '07

26 Mar '07

Update of /project/elephant/cvsroot/elephant/doc In directory clnet:/tmp/cvs-serv18053 Modified Files: tutorial.texinfo user-guide.texinfo Log Message: First cut new tutorial --- /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/03/25 11:04:38 1.8 +++ /project/elephant/cvsroot/elephant/doc/tutorial.texinfo 2007/03/26 03:37:27 1.9 @@ -355,16 +355,9 @@ Using the @code{persistent-metaclass} metaclass declares all slots to be persistent by default. To make a non-persistent slot use the -@code{:transient t} flag. Class slots are never persisted, for either -persistent or ordinary classes. (Someday, if we choose to store class -objects, this policy decision may change). - -Readers, writers, accessors, and @code{slot-value-using-class} are -instrumented, so override these with care. Because @code{slot-value, -slot-unboundp, slot-makunbound} are not generic functions, they are -not guaranteed to work properly with persistent slots. Use the -@code{*-using-class} versions or the @code{closer-to-mop} MOP compliance -layer by Pascal Costanza (we may integrate this in later versions). +@code{:transient t} flag. Class slots @code{:allocation :class} are +never persisted, for either persistent or ordinary classes. (Someday, +if we choose to store class objects, this policy may change). Persistent classes may inherit from other classes. Slots inherited from persistent classes remain persistent. Transient slots and slots @@ -372,47 +365,105 @@ cannot inherit from persistent classes -- otherwise persistent slots could not be stored! -Note that the database is read every time you access a slot. This is -a feature, not a bug, especially in concurrent situations: you want -the most recent commits, right? Note that this can be used as a weak -form of IPC. But also note that in particular, if your slot value is -not an immediate value, reading will cons or allocate the value. Gets -are not an expensive operation; you can perform thousands to tens of -thousands of primitive reads per second. However, if you're -concerned, cache large values in memory. +@lisp +(defclass stdclass1 () + ((slot1 :initarg :slot1 :accessor slot1))) + +(defclass stdclass2 (stdclass1) + ((slot2 :initarg :slot2 :accessor slot2))) + +(defpclass pclass1 (stdclass2) + ((slot1 :initarg :slot1 :accessor slot1) + (slot3 :initarg :slot3 :accessor slot3))) + +(make-instance 'pclass1 :slot1 1 :slot2 2 :slot3 3) +=> #<PCLASS1 @{x10deb88a@}> + +(add-to-root 'pinst *) +=> #<PCLASS1 @{x10deb88a@}> + +(slot1 pinst) +=> 1 + +(slot2 pinst) +=> 2 + +(slot3 pinst) +=> 3 +@end lisp + +Now we can simulate a new lisp session by flushing the instance cache, +reloading our object then see what slots remain. Here persistent +slot1 should shadow the standard slot1 and thus be persistent. Slot3 +is persistent by default and slot2, since it is inherited from a +standard class should be transient. + +@lisp +(elephant::flush-instance-cache *store-controller*) +=> #<EQL hash-table with weak values, 0 entries @{x11198a02@}> + +(setf pinst (get-from-root 'pinst)) +=> #<PCLASS1 @{x1119b652@}> + +(slot1 pinst) +=> 1 + +(slot-boundp pinst slot2 pinst) +=> nil + +(slot3 pinst) +=> 3 +@end lisp + +Using persistent objects has implications for the performance of your +system. Note that the database is read every time you access a slot. +This is a feature, not a bug, especially in concurrent situations: you +want the most recent commits by other threads, right? This can be +used as a weak form of IPC. But also note that in particular, if your +slot value is not an immediate value or persistent object, reading +will cons or freshly allocate storage for the value. + +Gets are not an expensive operation; you can perform thousands to tens +of thousands of primitive reads per second. However, if you're +concerned, cache large values in memory and avoid writing them back to +disk as long as you can. @node Persistent collections @comment node-name, next, previous, up @section Persistent collections -The remaining problem outlined in @ref{Serialization} is that -operations which mutate aggregate objects are not persistent. While -we solved this problem for objects, there is no collection type such -as arrays, hashes or lists which provide this ability. Elephant -provides two primary types of collections, a @code{btree} and a -@code{indexed-btree}. - -We will focus on the core concepts of BTrees in this section, for a -detailed review including the behavior of indexed BTrees, @pxref{Using -BTrees}, @ref{Secondary Indices} and @ref{Using Cursors} in the -@ref{User Guide}. - -Elephant provides a rich data structure called a BTree for storing -large sets of key-value pairs. Every key-value pair is stored -independantly in Elephant just like persistent object slots. -Therefore they inherit all the nice properties of persistent objects: -identity, fast serialization / deserialization, no merge conflicts, -etc. +The remaining problem outlined in the section on @ref{Serialization} +is that operations which mutate collection types do not have +persistent side effects. We have solved this problem for objects, but +not for collections such as as arrays, hashes or lists. Elephant's +solution to this problem is the @code{btree} class which provides +persistent addition, deletion and mutation of elements. + +The BTree stores arbitrarily sized sets of key-value pairs ordered by +key. Every key-value pair is stored independantly in Elephant just +like persistent object slots. They inherit all the important +properties of persistent objects: btree identity and fast +serialization / deserialization. They also resolve the mutated +substructure and nested aggregates problem for collections. Every +mutating write to a btree is an independent and persistent operation +and you can serialize or deserialize a btree without serializing any +of it's key-value pairs. The primary interface to @code{btree} objects is through -@code{get-value}. You can also @code{setf} @code{get-value} to store -key-value pairs. +@code{get-value}. You use @code{setf} @code{get-value} to store +key-value pairs. This interface is very similar to @code{gethash}. + +The following example creates a btree called +@code{*friends-birthdays*} and adds it to the root so we can retrieve +it during a later sessions. We then will add two key-value pairs +consisting of the name of a friend and a universal time encoding their +birthday. @lisp (defvar *friends-birthdays* (make-btree)) => *FRIENDS-BIRTHDAYS* -(add-to-root "friends-birthdays" *friends-birthdays*) +(add-to-root 'friends-birthdays *friends-birthdays*) => #<BTREE @{4951CF6D@}> (setf (get-value "Ben" *friends-birthdays*) @@ -445,8 +496,8 @@ due to serialization semantics may be strange for other values like arrays, lists, standard-objects, etc. -Because elements are sorted by value, we should be able to iterate -over all the elements of the BTree in order. We entered the data in +Because elements are sorted by value, we can iterate over all the +elements of the BTree in order. Notice that we entered the data in reverse alphabetic order, but will read it out in alphabetical order. @lisp @@ -459,16 +510,21 @@ => NIL @end lisp -But what if we want to read out our friends from oldest to youngest, -or youngest to oldest? In the @ref{User Guide}, specifically the -section on @ref{Secondary indices} you will discover ways to sort -according to the order defined by a lisp function of the key-value pair. +But what if we want to read out our friends from oldest to youngest? +One way is to employ another btree that maps birthdays to names, but +this will require storing values multiple times for each update and +increases the burden on the programmer. Elephant provides a better +way. + +The next section @ref{Indexing Persistent Classes} shows you how to +order and retrieve persistent classes by one or more slot values. + @node Indexing Persistent Classes @comment node-name, next, previous, up @section Indexing Persistent Classes -Class indices simplify the recording and retrieving of persistent +Class indexing simplifies the storing and retrieval of persistent objects. An indexed class stores every instance of the class that is created, ensuring that every object is automatically persisted between sessions. @@ -571,16 +627,20 @@ increase the cost of writes and disk storage, each entry is only slightly larger than the size of the slot value. Numbers, small strings and symbols are good candidate types for indexed slots, but -any value may be used, even different types. +any value may be used, even different types. Once a slot is indexed, +we can use the index to retrieve objects by slot values. -Once we've indexed a slot, we can use another set of -@code{get-instances} and @code{map} functions to access objects -in-order and by their slot value. +@code{get-instances-by-value} will retrieve all instances that are +equal to the value argument. @lisp (get-instances-by-value 'friends 'name "Carlos") => (#<Carlos>) +@end lisp +But more interestingly, we can retrieve objects for a range of values. + +@lisp (get-instances-by-range 'friends 'name "Adam" "Devin") => (#<Adriana> #<Carlos>) @@ -591,78 +651,67 @@ name: Zaid birthdate: (14 8 1976) name: Adriana birthdate: (24 4 1980) => (#<Zaid> #<Adriana>) +@end lisp + +To retrieve all instances of a class in the order of the index instead +of the arbitrary order returned by @code{get-instances-by-class} you +can use nil in the place of the start and end values to indicate the +first or last element. (Note: to retrieve instances null values, use +@code{get-instances-by-value} with nil as the argument). -(map-class-index #'print-friend 'friend 'name "Carlos" "Carlos") +@lisp +(get-instances-by-range 'friend 'name nil "Sandra") +=> (#<Adriana> #<Carlos>) + +(get-instances-by-range 'friend 'name nil nil) +=> (#<Adriana> #<Carlos> #<Zaid>) +@end lisp + +There are also functions for mapping over instances of a slot index. +To map over values, use the :value keyword argument. To map by range, +use the :start and :end arguments. + +@lisp +(map-class-index #'print-friend 'friend 'name :value "Carlos") name: Carlos birthdate: (1 1 1972) => NIL -(map-class-index #'print-friend 'friend 'name "Adam" "Devin") +(map-class-index #'print-friend 'friend 'name :start "Adam" :end "Devin") name: Adriana birthdate: (24 4 1980) name: Carlos birthdate: (1 1 1972) => NIL (map-class-index #'print-friend 'friend 'birthday - (encode-birthday '(1 1 1974)) - (encode-birthday '(31 12 1984))) + :start (encode-birthday '(1 1 1974)) + :end (encode-birthday '(31 12 1984))) name: Zaid birthdate: (14 8 1976) name: Adriana birthdate: (24 4 1980) => NIL -(map-class-index #'print-friend 'friend 'birthday nil (encode-birthday '(10 10 1978))) +(map-class-index #'print-friend 'friend 'birthday + :start nil + :end (encode-birthday '(10 10 1978))) name: Carlos birthdate: (1 1 1972) name: Zaid birthdate: (14 8 1976) => NIL (map-class-index #'print-friend 'friend 'birthday - (encode-birthday '(10 10 1975)) - nil) + :start (encode-birthday '(10 10 1975)) + :end nil) name: Zaid birthdate: (14 8 1976) name: Adriana birthdate: (24 4 1980) => NIL @end lisp -You can enable/disable class indexing for an entire class. When you disable -indexing all references to instances of that class are lost. If you re-enable -class indexing only newly created classes will be stored in the class index. -You can manually restore them by using @code{find-class-index} to get the -clas index BTree if you have an alternate in-memory index. - -You can add/remove a secondary index for a slot. So long as the class index -remains, this can be done multiple times without losing any data. - -There is also a facility for defining 'derived slots'. These can be non-slot -parameters which are a function of the class's persistent slot values. For -example you can use an index to keep an alternate representation available -for fast indexing. If an object has an x,y coordinate, you could define a -derived index for r,theta which stored references in polar coordinates. -These would be ordered so you could iterate over a class-index to get objects -in order of increasing radius from the origin or over a range of theta. - -Beware, however, that derived indices have to compute their result every -time you update any persistent instance's slot. This is because there is -no way to know which persistent slots the derived index value(s) depends -on. Thus there is a fairly significant computational cost to objects -with frequent updates having derived indices. The storage cost, however, -may be less as all that is added is the index value and an OID reference -into the class index. To add a slot value you add a serialized -OID+class-ref+slotname to index value which can be much larger if you -use long slotnames and package names and unicode. - -Thus, the question of if and how a given class should be indexed is -very flexible and dynamic, and does not need to be determined at the -beginning of your development. This represents the ability to ``late bind'' -the decision of what to index. - -In general, there is always a tradeoff: an indexed slot increases storage -associated with that slot and slows down write operations. Reads however remain -as fast as for unindexed persistent slots. The Elephant system -makes it simple to choose where and when one wants to utilize this tradeoff. - -Finally, that file @file{src/elephant/classindex-utils.lisp} documents -tools for handling class redefinitions and the policy that should be -used for synchronizing the classes with the database. This process is -somewhat user customizable; documentation for this exists in the source -file referenced above. +The @ref{User Guide} contains a descriptions of the advanced features +of @ref{Class indices} such as ``derived indicies'' that allow you to +order classes according to an arbitrary function, a dynamic API for +adding and removing slots and how to set a policy for resolving +conflicts between the code image and a database where the indexing +specification differs. + +This same facility is also available for your own use. For more +information @pxref{Using Indexed BTrees}. @node Using Transactions @@ -670,24 +719,24 @@ @section Using Transactions One of the most important features of a database is that operations -satisfy the ACID properties: Atomic, Consistent, Isolated, and +enforce the ACID properties: Atomic, Consistent, Isolated, and Durable. In plainspeak, this means that a set of changes is made all at once, that the database is never partially updated, that each set of changes happens sequentially and that a change, once made, is not lost. Elephant provides this protection for all primitive operations. For -example, when you write a value to an indexed BTree, the update to the -BTree and all of its indices is protected by a transaction that -peforms atomic updates to all the BTrees, thus maintaining their -consistency. - -Most real applications will need to have explicit transactions because -you will want one or more read-modify-update operations to happen as -an atomic unit. A common motivating example for this is a banking -system. If a thread is going to modify a balance, we don't want -another thread modifying it in the middle of the operation or one of -the modifications may be lost. +example, when you write a value to an indexed slot, the update to the +persistent slot record as well as the slot index is protected by a +transaction that performs all the updates atomically and thus +enforcing consistency. + +Most real applications will need to use explicit transactions rather +than relying on the primitives alone because you will want multiple +read-modify-update operations act as an atomic unit. A good example +for this is a banking system. If a thread is going to modify a +balance, we don't want another thread modifying it in the middle of +the operation or one of the modifications may be lost. @lisp (defvar *accounts* (make-btree)) --- /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/03/25 11:04:38 1.2 +++ /project/elephant/cvsroot/elephant/doc/user-guide.texinfo 2007/03/26 03:37:27 1.3 @@ -34,6 +34,12 @@ @code{initforms} are always evaluated, so beware. (What is the current model here?) +Readers, writers, accessors, and @code{slot-value-using-class} are +employed in redirecting slot accesses to the database, so override +these with care. Because @code{slot-value, slot-boundp, +slot-makunbound} are not generic functions, they are not guaranteed by +the specification to work properly with persistent slots. However the +proper behavior has been verified on SBCL, Allegro and Lispworks. @node The Store Controller @comment node-name, next, previous, up

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 25 Mar '07

25 Mar '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv32631/elephant Modified Files: classindex.lisp collections.lisp Log Message: Another fix for map-index / map-class-index and adding ranges for map-btree (but not map-class --- /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/03/24 12:16:03 1.32 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/03/25 14:57:49 1.33 @@ -373,10 +373,12 @@ (declare (dynamic-extent map-fn)) (map-btree #'map-fn class-idx)))) -(defun map-class-index (fn class index start end) - "If you want to map over a subset of instances, pick an index - and specify bounds for the traversal. Otherwise use map-class - for all instances" +(defun map-class-index (fn class index &rest args &key start end value) + "To map over a subset of instances, pick an index by slot name + or derived index name and specify the bounds for the traversal. + Otherwise use map-class for all instances. " + (declare (dynamic-extent args) + (ignorable args)) (let* ((index (if (symbolp index) (find-inverted-index class index) index))) @@ -384,7 +386,7 @@ (declare (ignore key pkey)) (funcall fn value))) (declare (dynamic-extent wrapper)) - (map-index #'wrapper index :start start :end end)))) + (map-index #'wrapper index :start start :end end :value value)))) ;; ================= @@ -426,8 +428,7 @@ (declare (ignore k pk)) (push v instances))) (declare (dynamic-extent collector)) - (map-index #'collector (find-inverted-index class slot-name) - :start value :end value)) + (map-index #'collector (find-inverted-index class slot-name) :value value)) (nreverse instances))) (defmethod get-instance-by-value ((class symbol) slot-name value) --- /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/03/23 16:18:59 1.18 +++ /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/03/25 14:57:49 1.19 @@ -318,11 +318,6 @@ different key.) Returns has-tuple / secondary key / value / primary key.")) - -;; ======================================= -;; Generic Mapping Functions -;; ======================================= - (defmacro with-btree-cursor ((var bt) &body body) "Macro which opens a named cursor on a BTree (primary or not), evaluates the forms, then closes the cursor." @@ -331,16 +326,9 @@ (progn ,@body) (cursor-close ,var)))) -(defmethod map-btree (fn (btree btree)) - "Like maphash. Default implementation - overridable - Function of two arguments key and value" - (ensure-transaction (:store-controller (get-con btree)) - (with-btree-cursor (curs btree) - (loop - (multiple-value-bind (more k v) (cursor-next curs) - (declare (dynamic-extent more k v)) - (unless more (return nil)) - (funcall fn k v)))))) +;; ======================================= +;; Generic Mapping Functions +;; ======================================= (defun lisp-compare<= (a b) (etypecase a @@ -348,15 +336,52 @@ (string (string<= a b)) (persistent (<= (oid a) (oid b))))) -(defun lisp-compare-eq (a b) - (eq a b)) +(defun lisp-compare-equal (a b) + (equal a b)) -(defmethod map-index (fn (index btree-index) &rest args &key start end) - "Like map-btree, but takes a function of three arguments key, value and primary key - if you want to get at the primary key value, otherwise use map-btree" +;; NOTE: the use of nil for the last element in a btree only works because the C comparison +;; function orders by type tag and nil is the highest valued type tag so nils are the last +;; possible element in a btree ordered by value. +(defmethod map-btree (fn (btree btree) &rest args &key start end (value nil value-set-p)) + "Map btree maps over a btree from the value start to the value of end. + If values are not provided, then it maps over all values. BTrees + do not have duplicates, but map-btree can also be used with indices + in the case where you don't want access to the primary key so we + require a value argument as well for mapping duplicate value sets." + (let ((end (if value-set-p value end))) + (ensure-transaction (:store-controller (get-con btree)) + (with-btree-cursor (curs btree) + (multiple-value-bind (exists? key value) + (cond (value-set-p + (cursor-set curs value)) + ((null start) + (cursor-first curs)) + (t (cursor-set-range curs start))) + (if exists? + (funcall fn key value) + (return-from map-btree nil)) + (loop + (multiple-value-bind (exists? k v) + (cursor-next curs) + (declare (dynamic-extent exists? k v)) + (if (and exists? (or (null end) (lisp-compare<= k end))) + (funcall fn k v) + (return nil))))))))) + +(defmethod map-index (fn (index btree-index) &rest args &key start end (value nil value-set-p)) + "Map-index is like map-btree but for secondary indices, it + takes a function of three arguments: key, value and primary + key. As with map-btree the keyword arguments start and end + determine the starting element and ending element, inclusive. + Also, start = nil implies the first element, end = nil implies + the last element in the index. If you want to traverse only a + set of identical key values, for example all nil values, then + use the value keyword which will override any values of start + and end." (declare (dynamic-extent args) (ignorable args)) - (let ((sc (get-con index))) + (let ((sc (get-con index)) + (end (or value end))) (ensure-transaction (:store-controller sc) (with-btree-cursor (cur index) (labels ((next-range () @@ -379,8 +404,8 @@ (next-range)))))) (declare (dynamic-extent next-range next-in-range)) (multiple-value-bind (exists? skey val pkey) - (cond ((lisp-compare-eq start end) - (cursor-pset cur start)) + (cond (value-set-p + (cursor-pset cur value)) ((null start) (cursor-pfirst cur)) (t (cursor-pset-range cur start))) @@ -393,7 +418,6 @@ nil))))))) - ;; =============================== ;; Some generic utility functions ;; ===============================

1 0