elephant-cvs February 2007

elephant-cvs@common-lisp.net

2 participants
167 discussions

[elephant-cvs] CVS elephant
by ieslick 16 Feb '07

16 Feb '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv28850 Modified Files: TODO Log Message: Cleaned up optimize-storage for BDB backends with optimize-layout generic function on stores and btrees --- /project/elephant/cvsroot/elephant/TODO 2007/02/16 03:43:47 1.49 +++ /project/elephant/cvsroot/elephant/TODO 2007/02/16 07:11:02 1.50 @@ -13,13 +13,10 @@ - Validate SQL migration 0.6.0->0.6.1 (Robert) - Legacy conversions issue for SQL (and BDB?) due to package rename (both) - TASKS TO GET TO BETA: BDB Features/Cleanup: - Always support locks that timeout? Tradeoffs? -- Figure out how to compact a specific btree and/or key-range using optimize-storage. - Probably need to update keyword part of the API - Perform checkpoints (prep for DCM functionality) - Verify db_deadlock for other lisps (launch and kill background program I/F) - Derived indices fail to re-connect, verify this @@ -63,7 +60,9 @@ 0.6.1 - Features COMPLETED to date ---------------------------------- -February 13th, 2007: +February 13-17th, 2007: +x Figure out how to compact a specific btree and/or key-range using optimize-storage. + Probably need to update keyword part of the API x User choice to run db_deadlock when opening a bdb backend? Requires path to functions and ability to launch shell command. Closing the store stops the sub-process. x Resolve duplicate ordering issues (punting to future release; documentation fix)

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 16 Feb '07

16 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv28850/src/db-bdb Modified Files: bdb-collections.lisp bdb-controller.lisp package.lisp Log Message: Cleaned up optimize-storage for BDB backends with optimize-layout generic function on stores and btrees --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-collections.lisp 2007/02/14 04:36:09 1.18 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-collections.lisp 2007/02/16 07:11:02 1.19 @@ -72,6 +72,13 @@ (db-delete-buffered (controller-btrees sc) key-buf)))) +(defmethod optimize-layout ((bt bdb-btree) &key (freelist-only t) (free-space nil) &allow-other-keys) + (optimize-layout (get-con bt) + :start-key (oid bt) + :end-key (oid bt) + :freelist-only freelist-only + :free-space free-space)) + ;; Secondary indices (defclass bdb-indexed-btree (indexed-btree bdb-btree) @@ -232,7 +239,7 @@ (buffer-write-oid (oid bt) key-buf) (serialize key key-buf (get-con bt)) (let ((buf (db-get-key-buffered - (controller-indices-assoc (get-con bt)) + (controller-indices-assoc (get-con bt)) key-buf value-buf))) (if buf (values (deserialize buf (get-con bt)) T) (values nil nil))))) --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/16 03:43:47 1.25 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/16 07:11:02 1.26 @@ -251,23 +251,29 @@ ;; Take advantage of release 4.4's compact storage feature. Feature of BDB only ;; -(defmethod optimize-storage ((ctrl bdb-store-controller) &key start-key stop-key - (freelist-only nil) (free-space t) - &allow-other-keys) +(defmethod optimize-layout ((ctrl bdb-store-controller) &key start-key stop-key + (freelist-only t) (free-space nil) + &allow-other-keys) "Tell the backend to optimize and reclaim storage between key values" (with-buffer-streams (start stop end) - (if (null start) + (if (null start-key) (progn - (db-compact (controller-db ctrl) nil nil end) - (db-compact (controller-btrees ctrl) nil nil end) (db-compact (controller-indices ctrl) nil nil end) - (db-compact (controller-indices-assoc ctrl) nil nil end) - (db-compact (controller-oid-db ctrl) nil nil end)) + (db-compact (controller-db ctrl) nil nil end) + (db-compact (controller-btrees ctrl) nil nil end)) (progn (serialize start-key start ctrl) - (db-compact (controller-db ctrl) start - (when stop-key (serialize stop-key stop ctrl) stop) - end + (when stop-key (serialize stop-key stop ctrl)) + (db-compact (controller-indices ctrl) start + (when stop-key stop) end + :freelist-only freelist-only + :free-space free-space) + (db-compact (controller-db ctrl) nil + (when stop-key stop) end + :freelist-only freelist-only + :free-space free-space) + (db-compact (controller-btrees ctrl) nil + (when stop-key stop) end :freelist-only freelist-only :free-space free-space))) (values (deserialize end ctrl)))) --- /project/elephant/cvsroot/elephant/src/db-bdb/package.lisp 2007/02/14 04:36:10 1.4 +++ /project/elephant/cvsroot/elephant/src/db-bdb/package.lisp 2007/02/16 07:11:02 1.5 @@ -41,4 +41,4 @@ (:import-from :ccl #:byte-length) (:export - #:optimize-storage)) + #:optimize-layout))

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 16 Feb '07

16 Feb '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv28850/src/elephant Modified Files: collections.lisp controller.lisp package.lisp Log Message: Cleaned up optimize-storage for BDB backends with optimize-layout generic function on stores and btrees --- /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/02/14 04:36:10 1.8 +++ /project/elephant/cvsroot/elephant/src/elephant/collections.lisp 2007/02/16 07:11:02 1.9 @@ -57,6 +57,9 @@ (defgeneric existsp (key bt) (:documentation "Test existence of a key / value pair in a BTree")) +(defmethod optimize-layout ((bt t) &key &allow-other-keys) + t) + ;; ;; Btrees that support secondary indices ;; --- /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/02/14 17:41:29 1.33 +++ /project/elephant/cvsroot/elephant/src/elephant/controller.lisp 2007/02/16 07:11:02 1.34 @@ -307,10 +307,11 @@ (:documentation "Provides a persistent source of unique id's")) -(defgeneric optimize-storage (sc &key &allow-other-keys) - (:documentation - "Tell the backend to reclaim any storage caused by key deletion, if possible. - This should default to return space to the filesystem rather than just to the free list.")) +(defgeneric optimize-layout (sc &key &allow-other-keys) + (:documentation "If supported, speed up the index and allocation by freeing up + any available storage and return it to the free list. See the + methods of backends to determine what options are valid. Supported + both on stores (all btrees and persistent slots) and specific btrees")) ;; ;; Low-level support for metaclass protocol @@ -377,7 +378,7 @@ ;; -;; Operations on the root index +; Operations on the root index ;; (defun add-to-root (key value &key (store-controller *store-controller*)) --- /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/02/14 17:41:29 1.13 +++ /project/elephant/cvsroot/elephant/src/elephant/package.lisp 2007/02/16 07:11:02 1.14 @@ -35,7 +35,7 @@ #:get-cached-instance #:flush-instance-cache #:controller-symbol-cache #:controller-symbol-id-cache #:controller-fast-symbols-p - #:optimize-storage + #:optimize-layout #:get-user-configuration-parameter #:upgrade

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 16 Feb '07

16 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv26996/src/db-bdb Modified Files: bdb-controller.lisp Log Message: Latest changes to launching deadlock processes, all lisps supported (but not tested) except lispworks --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/14 17:41:29 1.24 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/16 03:43:47 1.25 @@ -232,29 +232,29 @@ (cdr result))) (defmethod start-deadlock-detector ((ctrl bdb-store-controller) &key (type :oldest) (time 0.1) log) - (multiple-value-bind (str errstr pid) - (launch-background-program - (second (controller-spec ctrl)) - (namestring - (make-pathname :defaults (get-user-configuration-parameter :berkeley-db-deadlock))) - :args `("-a" ,(lookup-deadlock-type type) - "-t" ,(format nil "~D" time) - ,@(when log (list "-L" (format nil "~A" log))))) + (let ((process-handle + (launch-background-program + (second (controller-spec ctrl)) + (namestring + (make-pathname :defaults (get-user-configuration-parameter :berkeley-db-deadlock))) + :args `("-a" ,(lookup-deadlock-type type) + "-t" ,(format nil "~D" time) + ,@(when log (list "-L" (format nil "~A" log))))))) (declare (ignore str errstr)) - (setf (controller-deadlock-pid ctrl) pid))) + (setf (controller-deadlock-pid ctrl) process-handle))) (defmethod stop-deadlock-detector ((ctrl bdb-store-controller)) (when (controller-deadlock-pid ctrl) (kill-background-program (controller-deadlock-pid ctrl)))) ;; -;; Take advantage of release 4.4's compact storage feature. Hidden features of BDB only +;; Take advantage of release 4.4's compact storage feature. Feature of BDB only ;; (defmethod optimize-storage ((ctrl bdb-store-controller) &key start-key stop-key (freelist-only nil) (free-space t) &allow-other-keys) - "Tell the backend to optimize storage between key values" + "Tell the backend to optimize and reclaim storage between key values" (with-buffer-streams (start stop end) (if (null start) (progn

1 0

[elephant-cvs] CVS elephant/src/utils
by ieslick 16 Feb '07

16 Feb '07

Update of /project/elephant/cvsroot/elephant/src/utils In directory clnet:/tmp/cvs-serv26996/src/utils Modified Files: os.lisp Log Message: Latest changes to launching deadlock processes, all lisps supported (but not tested) except lispworks --- /project/elephant/cvsroot/elephant/src/utils/os.lisp 2007/02/14 04:38:56 1.1 +++ /project/elephant/cvsroot/elephant/src/utils/os.lisp 2007/02/16 03:43:48 1.2 @@ -1,26 +1,54 @@ (in-package :elephant-utils) +(defmacro in-directory ((dir) &body body) + `(progn + (#+sbcl sb-posix:chdir + #+cmu unix:unix-chdir + #+allegro excl:chdir + #+lispworks hcl:change-directory + #+openmcl ccl:cwd + ,dir) + ,@body)) + (defun launch-background-program (directory program &key (args nil)) "Launch a program in a specified directory - not all shell interfaces or OS's support this" #+(and allegro (not mswindows)) - (excl:run-shell-command (concat-separated-strings " " (list program) args) - :wait nil - :directory directory) -;; #+(and allegro mswindows) -;; #+(and sbcl unix) -;; (sb-ext:start-process ... -;; #+(and openmcl unix) -;; #+lispworks + (multiple-value-bind (in out pid) + (excl:run-shell-command (concat-separated-strings " " (list program) args) + :wait nil + :directory directory) + (declare (ignore in out)) + pid) + #+(and sbcl unix) + (in-directory (directory) + (sb-ext:run-program program args :wait nil)) + #+cmu + (in-directory (directory) + (ext:run-program program args :wait nil)) + #+openmcl + (in-directory (directory) + (ccl:run-program program args :wait nil)) + #+lispworks + (apply #'sys::call-system + (format nil "~a~{ '~a'~}~@[ &~]" prog args) + :current-directory directory + :wait nil) ) -(defun kill-background-program (pid) +(defun kill-background-program (process-handle) #+(and allegro (not mswindows)) - (progn (excl.osi:kill pid 9) - (system:reap-os-subprocess :pid pid)) -;; #+(and allegro mswindows) + (progn (excl.osi:kill process-handle 9) + (system:reap-os-subprocess :pid process-handle)) #+(and sbcl unix) - (sb-ext:process-kill "/bin/kill" (list "-9" (format nil "~A" pid))) + (sb-ext:process-kill process-handle 9) + #+openmcl + (ccl:signal-external-process process-handle 9) +;; #+lispworks +;; (apply #'sys::call-system +;; (format nil "kill ~A -9" process-handle) +;; :current-directory directory +;; :wait t) )

1 0

[elephant-cvs] CVS elephant
by ieslick 16 Feb '07

16 Feb '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv26996 Modified Files: TODO elephant.asd Log Message: Latest changes to launching deadlock processes, all lisps supported (but not tested) except lispworks --- /project/elephant/cvsroot/elephant/TODO 2007/02/14 04:36:08 1.48 +++ /project/elephant/cvsroot/elephant/TODO 2007/02/16 03:43:47 1.49 @@ -227,6 +227,8 @@ (ala AllegroCache) - Support for cheap persistent sets (medium? can do on SQL?) - Native persistent hashes (easy for BDB; can do on SQL backends?) + - Persistent aggregates for better conceptual integration with lisp? + - pcons, parray, pstruct, etc - Support a simple object query language over the database - Repository browser - a simple REPL tool like the Slime inspector to see what classes are in a repository and what state they're in...useful --- /project/elephant/cvsroot/elephant/elephant.asd 2007/02/14 04:36:08 1.29 +++ /project/elephant/cvsroot/elephant/elephant.asd 2007/02/16 03:43:47 1.30 @@ -161,7 +161,8 @@ ((:file "package") (:file "convenience") (:file "locks") - (:file "os"))) + (:file "os")) + :serial t) (:module memutil :components ((:elephant-c-source "libmemutil")

1 0

[elephant-cvs] CVS elephant/src/db-clsql
by rread 14 Feb '07

14 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-clsql In directory clnet:/tmp/cvs-serv3373 Modified Files: sql-controller.lisp Log Message: Changes to make sqlite3 work --- /project/elephant/cvsroot/elephant/src/db-clsql/sql-controller.lisp 2007/02/08 22:33:35 1.18 +++ /project/elephant/cvsroot/elephant/src/db-clsql/sql-controller.lisp 2007/02/14 22:39:10 1.19 @@ -41,6 +41,13 @@ creation, counters, locks, the root (for garbage collection,) et cetera. This is the Postgresql-specific subclass of store-controller.")) +(defmethod supports-sequence ((sc sql-store-controller)) + (not (equal + (car (cadr (controller-spec sc))) + :sqlite3) + ) +) + ;; This should be much more elegant --- but as of Feb. 6, SBCL 1.0.2 has a weird, ;; unpleasant bug when ASDF tries to load this stuff. ;; (defvar *thread-table-lock* nil) @@ -52,13 +59,13 @@ (if (null *thread-table-lock*) ;; nil ;; (setq *thread-table-lock* (sb-thread::make-mutex :name "thread-table-lock")) - (setq *thread-table-lock* (elephant::ele-make-lock)) + (setq *thread-table-lock* (elephant-utils::ele-make-lock)) ) ) (defun thread-hash () - (elephant::ele-thread-hash-key) + (elephant-utils::ele-thread-hash-key) ) @@ -284,14 +291,26 @@ (clsql:table-exists-p [version] :database con :owner :all) ) -(defun create-version-table (con) - ;; ALL OF THIS needs to be inside a transaction. - (clsql::create-table [version] +(defun create-version-table (sc) + (let ((con (controller-db sc))) + (clsql::create-table [version] '( ([dbversion] text :not-null) ) :database con ) - ) + (if (and (clsql:table-exists-p [keyvlaue] :database con :owner :all) + (= 0 (caar (clsql:query "select count(*) from keyvalue")))) + (clsql::insert-records :into [version] + :attributes '(dbversion) + :values (list (format nil "~A" *elephant-code-version*)) + :database con) + (clsql::insert-records :into [version] + :attributes '(dbversion) + :values (list (format nil "~A" '(0 6 0))) + :database con) + ) + ) + ) ;; These functions are probably not cross-database portable... (defun keyvalue-table-exists (con) @@ -320,7 +339,7 @@ ;; storing blobs now in the Berkeley-db and we meed to make sure ;; we are properly testing that. However, blobs are awkward to ;; handle, so I am going to do this first... -(defun create-keyvalue-table (con) +(defun create-keyvalue-table (sc) ;; the "serial" specifiation here does not seem to work, ( ;; apparently not supported by clsql, so I have to execute these ;; commands specifically. This may be a database-dependent way of doing @@ -334,25 +353,29 @@ ;; CREATE-SEQUENCE and SEQUENCE-NEXT. That would solve our problem! ;; ALL OF THIS needs to be inside a transaction. - + (let* ((con (controller-db sc))) ;; At one time this was conditional, but all NEW repositories should have this. - (clsql::create-sequence [serial] :database con) - (clsql::query - (format nil "create table keyvalue ( + (if (supports-sequence sc) + (progn + (clsql::create-sequence [serial] :database con) + (clsql::query + (format nil "create table keyvalue ( pk integer PRIMARY KEY DEFAULT nextval('serial'), clctn_id integer NOT NULL, key varchar NOT NULL, value varchar )") - :database con) -;; (clsql::create-table [keyvalue] -;; ;; This is most likely to work with any database system.. -;; '( -;; ([clctn_id] integer :not-null) -;; ([key] text :not-null) -;; ([value] text) -;; ) -;; :database con) + :database con) + ) + (clsql::create-table [keyvalue] + ;; This is most likely to work with any database system.. + '( + ([clctn_id] integer :not-null) + ([key] text :not-null) + ([value] text) + ) + :database con) + ) ;; :constraints '("PRIMARY KEY (clctn_id key)" ;; "UNIQUE (clctn_id,key)") @@ -380,7 +403,7 @@ :attributes '([clctn_id] [key]) :database con) ;;) - ) + )) (defmethod database-version ((sc sql-store-controller)) "A version determination for a given store @@ -397,13 +420,6 @@ (read-from-string (caar tuples)) nil)))) -(defun set-database-version (sc) - (let ((con (controller-db sc))) - (clsql::insert-records :into [version] - :attributes '(dbversion) - :values (list (format nil "~A" *elephant-code-version*)) - :database con))) - (defmethod open-controller ((sc sql-store-controller) ;; At present these three have no meaning @@ -427,13 +443,11 @@ ;; it does not, we need to create it.. (unless (keyvalue-table-exists con) (with-transaction (:store-controller sc) - (create-keyvalue-table con))) + (create-keyvalue-table sc))) (setf (uses-pk-of sc) (query-uses-pk con)) (unless (version-table-exists con) (with-transaction (:store-controller sc) - (create-version-table con))) - ;; Set elephant version if new - (when new-p (set-database-version sc)) + (create-version-table sc))) (initialize-serializer sc) ;; These should get oid 0 and 1 respectively (setf (slot-value sc 'root) (make-instance 'sql-btree :sc sc :from-oid 0))

1 0

[elephant-cvs] CVS elephant/src/contrib/eslick/db-lisp
by ieslick 14 Feb '07

14 Feb '07

Update of /project/elephant/cvsroot/elephant/src/contrib/eslick/db-lisp In directory clnet:/tmp/cvs-serv23445/src/contrib/eslick/db-lisp Added Files: tests.lisp Log Message: Missing files for in-development backends --- /project/elephant/cvsroot/elephant/src/contrib/eslick/db-lisp/tests.lisp 2007/02/14 17:42:23 NONE +++ /project/elephant/cvsroot/elephant/src/contrib/eslick/db-lisp/tests.lisp 2007/02/14 17:42:23 1.1 ;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Base: 10 -*- ;;; ;;; test.lisp -- Internal tests for elephant lisp backend ;;; ;;; part of ;;; ;;; Elephant Object Oriented Database: Common Lisp Backend ;;; ;;; Copyright (c) 2007 by Ian Eslick ;;; <ieslick at common-lisp.net> ;;; ;;; Elephant Lisp Backend users are granted the rights to distribute ;;; and use this software as governed by the terms of the Lisp Lesser ;;; GNU Public License (http://opensource.franz.com/preamble.html), ;;; also known as the LLGPL. (in-package :db-lisp) (defparameter *btree-file* "btree-test-file.db") (defvar btree nil) (defvar root nil) (defvar idx1 nil) (defvar idx2 nil) (defvar leaf1 nil) (defvar leaf2 nil) (defun initialize-btree-tests () (setf btree (open-lisp-btree *btree-file* :if-exists :overwrite)) (setf root (btree-root btree)) btree) (defun cleanup-btree-tests () (close-lisp-btree btree) (setf root nil idx1 nil idx2 nil leaf1 nil leaf2 nil)) (defun dump-keys (page) (with-buffer-streams (k) (scan-page-keys (k pointer position btree page) (format t "k: ~A ptr: ~A pos: ~A~%" (elephant-serializer2::deserialize k nil) pointer position)))) (defmacro with-key ((bs num) &body body) `(with-buffer-stream (,bs) (elephant-serializer2::serialize ,num ,bs) ,@body))

1 0

[elephant-cvs] CVS elephant/src/contrib/eslick/db-acache
by ieslick 14 Feb '07

14 Feb '07

Update of /project/elephant/cvsroot/elephant/src/contrib/eslick/db-acache In directory clnet:/tmp/cvs-serv23445/src/contrib/eslick/db-acache Added Files: ele-acache.asd Log Message: Missing files for in-development backends --- /project/elephant/cvsroot/elephant/src/contrib/eslick/db-acache/ele-acache.asd 2007/02/14 17:42:22 NONE +++ /project/elephant/cvsroot/elephant/src/contrib/eslick/db-acache/ele-acache.asd 2007/02/14 17:42:22 1.1 ;;; -*- Mode: Lisp; Syntax: ANSI-Common-Lisp; Base: 10 -*- ;;; ;;; ele-acache.asd -- ASDF file for allegrocache backend ;;; ;;; Initial version 2/18/2006 by Ian Eslick ;;; <ieslick(a)common-lisp.net> ;;; ;;; part of ;;; ;;; Elephant: an object-oriented database for Common Lisp ;;; ;;; Copyright (c) 2004 by Andrew Blumberg and Ben Lee ;;; <ablumberg(a)common-lisp.net> <blee(a)common-lisp.net> ;;; ;;; Elephant users are granted the rights to distribute and use this software ;;; as governed by the terms of the Lisp Lesser GNU Public License ;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. (defsystem ele-acache :name "elephant" :author "Ben Lee <blee(a)common-lisp.net>" :version "0.6.0" :maintainer "Ben Lee <blee(a)common-lisp.net>" :licence "LLGPL" :description "Allegro cache backend for elephant" :components ((:module :src :components ((:module :db-acache :components ((:file "package") (:file "acache-controller") (:file "acache-transactions") (:file "acache-collections")) :serial t)))) :depends-on (:elephant))

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 14 Feb '07

14 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv23282/src/db-bdb Modified Files: bdb-controller.lisp Log Message: User parameter if; deadlock detector param support --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/14 04:36:10 1.23 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/14 17:41:29 1.24 @@ -236,8 +236,7 @@ (launch-background-program (second (controller-spec ctrl)) (namestring - (make-pathname :directory '(:ABSOLUTE "opt" "local" "bin" "db45_deadlock") - :name "db_deadlock")) + (make-pathname :defaults (get-user-configuration-parameter :berkeley-db-deadlock))) :args `("-a" ,(lookup-deadlock-type type) "-t" ,(format nil "~D" time) ,@(when log (list "-L" (format nil "~A" log)))))

1 0

← Newer
1
...
4
5
6
7
8
9
10
...
17
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

2004

elephant-cvs February 2007