elephant-cvs February 2007

elephant-cvs@common-lisp.net

2 participants
167 discussions

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 20 Feb '07

20 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv7106a/db-bdb Modified Files: bdb-transactions.lisp Log Message: Enable with-transaction to return multiple values in both backends --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/02/16 23:02:51 1.8 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-transactions.lisp 2007/02/20 02:33:59 1.9 @@ -45,7 +45,7 @@ (declare (special *current-transaction* *store-controller*)) (catch 'transaction (unwind-protect - (prog1 + (multiple-value-prog1 (funcall txn-fn) (db-transaction-commit txn :txn-nosync txn-nosync :txn-sync txn-sync)

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 19 Feb '07

19 Feb '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv9012 Modified Files: classes.lisp classindex-utils.lisp classindex.lisp Log Message: Fix reconnect to derived index bug under :class synchronization policy --- /project/elephant/cvsroot/elephant/src/elephant/classes.lisp 2007/02/14 04:36:10 1.13 +++ /project/elephant/cvsroot/elephant/src/elephant/classes.lisp 2007/02/18 23:38:18 1.14 @@ -130,14 +130,14 @@ (let* ((class (find-class (class-name (class-of instance)))) (oid (oid instance)) (persistent-slot-names (persistent-slot-names class))) - (flet ((persistent-slot-p (item) + (flet ((persistent-slot-p (item) (member item persistent-slot-names :test #'eq))) (let ((transient-slot-inits (if (eq slot-names t) ; t means all slots (transient-slot-names class) (remove-if #'persistent-slot-p slot-names))) (persistent-slot-inits - (if (eq slot-names t) + (if (eq slot-names t) persistent-slot-names (remove-if-not #'persistent-slot-p slot-names)))) (inhibit-indexing oid) --- /project/elephant/cvsroot/elephant/src/elephant/classindex-utils.lisp 2007/02/02 23:51:58 1.4 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex-utils.lisp 2007/02/18 23:38:18 1.5 @@ -12,6 +12,8 @@ (in-package :elephant) +(declaim #-elephant-without-optimize (optimize (speed 3) (safety 1))) + ;; ;; Simple utilities for managing synchronization between class ;; definitions and database state @@ -226,7 +228,6 @@ (simple-match-set (synch-rule-lhs rule) features)) (defun simple-match-set (a b) - (declare (optimize (speed 3) (safety 1))) (cond ((null a) t) ((and (not (null a)) (null b)) nil) ((member (first a) b :test #'equal) @@ -252,7 +253,6 @@ (warn (warn "Performing slot synchronization actions: ~A" (synch-rule-rhs rule)))))) (defun apply-synch-rules (class records rule-set) - (declare (optimize (speed 3) (safety 1))) (labels ((slotname (rec) (car rec)) (feature-set (rec) (cdr rec))) (loop for record in records do @@ -267,8 +267,7 @@ (let* ((*store-controller* store-controller) ;; db info (db-indices (find-inverted-index-names class)) - (db-derived (mapcar #'get-derived-name-root - (remove-if-not #'derived-name? db-indices))) + (db-derived (remove-if-not #'derived-name? db-indices)) (db-slot (set-difference db-indices db-derived)) ;; class info (marked-slots (indexing-record-slots (indexed-record class))) --- /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/02/14 04:36:10 1.17 +++ /project/elephant/cvsroot/elephant/src/elephant/classindex.lisp 2007/02/18 23:38:18 1.18 @@ -266,7 +266,8 @@ (defmethod remove-class-slot-index ((class symbol) slot-name &key (sc *store-controller*)) (remove-class-slot-index (find-class class) slot-name :sc sc)) -(defmethod remove-class-slot-index ((class persistent-metaclass) slot-name &key (sc *store-controller*) (update-class t)) +(defmethod remove-class-slot-index ((class persistent-metaclass) slot-name &key + (sc *store-controller*) (update-class t)) ;; NOTE: Write routines to recover BDB storage when you've wiped an index... ;; NOTE: If the transaction aborts we should not update class slots? (if (find-inverted-index class slot-name :null-on-fail t) @@ -282,7 +283,8 @@ (defmethod add-class-derived-index ((class symbol) name derived-defun &key (sc *store-controller*) (populate t)) (add-class-derived-index (find-class class) name derived-defun :sc sc :populate populate)) -(defmethod add-class-derived-index ((class persistent-metaclass) name derived-defun &key (populate t) (sc *store-controller*) (update-class t)) +(defmethod add-class-derived-index ((class persistent-metaclass) name derived-defun &key + (populate t) (sc *store-controller*) (update-class t)) (let ((class-idx (find-class-index class :sc sc))) (if (find-inverted-index class (make-derived-name name) :null-on-fail t) (error "Duplicate derived index requested named ~A on class ~A" name (class-name class)) @@ -297,7 +299,8 @@ (defmethod remove-class-derived-index ((class symbol) name &key (sc *store-controller*)) (remove-class-derived-index (find-class class) name :sc sc)) -(defmethod remove-class-derived-index ((class persistent-metaclass) name &key (sc *store-controller*) (update-class t)) +(defmethod remove-class-derived-index ((class persistent-metaclass) name &key + (sc *store-controller*) (update-class t)) (if (find-inverted-index class name :null-on-fail t) (progn (when update-class (unregister-derived-index class name))

1 0

[elephant-cvs] CVS elephant
by ieslick 18 Feb '07

18 Feb '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv392 Modified Files: elephant.asd Log Message: Fixed typo (again:) that Henrik pointed out --- /project/elephant/cvsroot/elephant/elephant.asd 2007/02/16 03:43:47 1.30 +++ /project/elephant/cvsroot/elephant/elephant.asd 2007/02/18 22:45:39 1.31 @@ -110,7 +110,7 @@ #-(or darwin macosx darwin-host) "-shared" #+(or darwin macosx darwin-host) "-bundle" #+(and X86-64 (or macosx darwin darwin-host)) "-arch x86_64" - #+(and X86-64 linux) "-march x86-64" + #+(and X86-64 linux) "-march=x86-64" "-Wall" "-fPIC" "-O3"

1 0

[elephant-cvs] CVS elephant/src/db-clsql
by ieslick 18 Feb '07

18 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-clsql In directory clnet:/tmp/cvs-serv26211 Modified Files: sql-controller.lisp Log Message: Fixed two typos --- /project/elephant/cvsroot/elephant/src/db-clsql/sql-controller.lisp 2007/02/18 21:08:16 1.20 +++ /project/elephant/cvsroot/elephant/src/db-clsql/sql-controller.lisp 2007/02/18 22:09:01 1.21 @@ -312,7 +312,7 @@ ) (sqlite3-harmless-read sc) (let ((version - (if (clsql:table-exists-p [keyvlaue] :database con :owner :all) + (if (clsql:table-exists-p [keyvalue] :database con :owner :all) (if (= 0 (caar (clsql:query "select count(*) from keyvalue"))) *elephant-code-version* '(0 6 0)) @@ -655,7 +655,7 @@ ) ) ) - (mapcar #'(lambda (x) (mapcar #'(lambda (q) (deserialize-from-base64-string q :sc sc)) x)) + (mapcar #'(lambda (x) (mapcar #'(lambda (q) (deserialize-from-base64-string q sc)) x)) tuples))) (defun sql-from-root-existsp (key sc)

1 0

[elephant-cvs] CVS elephant/src/db-clsql
by rread 18 Feb '07

18 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-clsql In directory clnet:/tmp/cvs-serv13893 Modified Files: sql-controller.lisp Log Message: Fixing kevalue typo, and trying to make SQLite3 warnings less painful. --- /project/elephant/cvsroot/elephant/src/db-clsql/sql-controller.lisp 2007/02/14 22:39:10 1.19 +++ /project/elephant/cvsroot/elephant/src/db-clsql/sql-controller.lisp 2007/02/18 21:08:16 1.20 @@ -290,27 +290,38 @@ ;; the table, as long as we have the rights we need! (clsql:table-exists-p [version] :database con :owner :all) ) - +(defun sqlite3-harmless-read (sc) + (let ((con (controller-db sc))) + (if + (equal + (car (cadr (controller-spec sc))) + :sqlite3) + (handler-case + (clsql:query "select count(*) from keyvalue") + ((clsql-sys::sql-database-error () nil) + ) + ) + ) + )) (defun create-version-table (sc) (let ((con (controller-db sc))) (clsql::create-table [version] - '( - ([dbversion] text :not-null) - ) :database con - ) - (if (and (clsql:table-exists-p [keyvlaue] :database con :owner :all) - (= 0 (caar (clsql:query "select count(*) from keyvalue")))) - (clsql::insert-records :into [version] - :attributes '(dbversion) - :values (list (format nil "~A" *elephant-code-version*)) - :database con) - (clsql::insert-records :into [version] - :attributes '(dbversion) - :values (list (format nil "~A" '(0 6 0))) - :database con) - ) - ) - ) + '( + ([dbversion] text :not-null) + ) :database con + ) + (sqlite3-harmless-read sc) + (let ((version + (if (clsql:table-exists-p [keyvlaue] :database con :owner :all) + (if (= 0 (caar (clsql:query "select count(*) from keyvalue"))) + *elephant-code-version* + '(0 6 0)) + *elephant-code-version*))) + (clsql::insert-records :into [version] + :attributes '(dbversion) + :values (list (format nil "~A" version)) + :database con) + ))) ;; These functions are probably not cross-database portable... (defun keyvalue-table-exists (con) @@ -358,6 +369,7 @@ (if (supports-sequence sc) (progn (clsql::create-sequence [serial] :database con) + (sqlite3-harmless-read sc) (clsql::query (format nil "create table keyvalue ( pk integer PRIMARY KEY DEFAULT nextval('serial'), @@ -376,7 +388,7 @@ ) :database con) ) - + (sqlite3-harmless-read sc) ;; :constraints '("PRIMARY KEY (clctn_id key)" ;; "UNIQUE (clctn_id,key)") @@ -402,6 +414,7 @@ (clsql:create-index [idx_both] :on [keyvalue] :attributes '([clctn_id] [key]) :database con) + (sqlite3-harmless-read sc) ;;) )) @@ -420,7 +433,6 @@ (read-from-string (caar tuples)) nil)))) - (defmethod open-controller ((sc sql-store-controller) ;; At present these three have no meaning &key

1 0

[elephant-cvs] CVS elephant
by ieslick 18 Feb '07

18 Feb '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv3903 Modified Files: TODO Log Message: Provide checkpoint function --- /project/elephant/cvsroot/elephant/TODO 2007/02/17 19:28:53 1.53 +++ /project/elephant/cvsroot/elephant/TODO 2007/02/18 10:58:58 1.54 @@ -14,8 +14,6 @@ TASKS TO GET TO BETA: BDB Features/Cleanup: -- Always support locks that timeout? Tradeoffs? -- Perform checkpoints (prep for DCM functionality) - Derived indices used to fail on re-connect, verify that this is fixed Lisp Support: @@ -49,22 +47,21 @@ Fix any bugs in BETA: Documentation: +- License and copyright file headers - Add document section about backend interface: - Add notes about with/ensure-transaction usage (abort & commit behavior on exit) - Add notes about optimize-storage - Add notes about deadlock-detect - Add notes about checkpoint (null in SQL?) -- Add notes about new BDB 4.5 *auto-commit* behavior. Default for entire - store-controller will auto create a transaction if none is active if open - with :auto-commit t or will never auto-commit (regardless of operator flags) - if it is not. Make sure open-store defaults to auto-commit and there is a - flag to turn it off. - More notes about transaction performance 0.6.1 - Features COMPLETED to date ---------------------------------- +February + February 13-17th, 2007: +x Allow checkpoint of BDB via db-bdb::checkpoint x Fix BDB migration 0.6.0->0.6.1, improve nested transaction/store interactions x Legacy conversions issue for SQL (and BDB?) due to package rename (both) x Figure out how to compact a specific btree and/or key-range using optimize-storage. @@ -173,6 +170,9 @@ (log these in Track) Features: + - Backup function: allow users to specify a backup function to archive the database contents + and checkpoint any active functions (how to lock out other threads?) In BDB this means + running checkpoint and copying the DB files and any active log files. - Persistent variables (abstraction that allows compound lisp objects at the cost of full serialization after each write that indirects through the API). Can this be done with clean semantics or should we punt it?

1 0

[elephant-cvs] CVS elephant/src/db-bdb
by ieslick 18 Feb '07

18 Feb '07

Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv3903/src/db-bdb Modified Files: bdb-controller.lisp package.lisp Log Message: Provide checkpoint function --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/17 19:28:53 1.29 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/18 10:58:58 1.30 @@ -258,7 +258,20 @@ (defmethod stop-deadlock-detector ((ctrl bdb-store-controller)) (when (controller-deadlock-pid ctrl) (kill-background-program (controller-deadlock-pid ctrl)))) + +;; +;; Enable program-based checkpointing +;; +(defmethod checkpoint ((sc bdb-store-controller) &key force (time 0) (log-size 0)) + "Forces a checkpoint of the db and flushes the memory pool to disk. + Use keywords ':force t' to write the checkpoint under any + condition, ':time N' to checkpoint based on if the number of + minutes since the last checkpoint is greater than time and + ':log-size N' to checkpoint if the data written to the log is + greater than N kilobytes" + (db-env-txn-checkpoint (controller-environment sc) log-size time :force force)) + ;; ;; Take advantage of release 4.4's compact storage feature. Feature of BDB only ;; --- /project/elephant/cvsroot/elephant/src/db-bdb/package.lisp 2007/02/16 23:02:51 1.6 +++ /project/elephant/cvsroot/elephant/src/db-bdb/package.lisp 2007/02/18 10:58:58 1.7 @@ -45,4 +45,5 @@ (:import-from :ccl #:byte-length) (:export - #:optimize-layout)) + #:optimize-layout + #:checkpoint))

1 0

[elephant-cvs] CVS elephant/src/elephant
by ieslick 18 Feb '07

18 Feb '07

Update of /project/elephant/cvsroot/elephant/src/elephant In directory clnet:/tmp/cvs-serv3903/src/elephant Modified Files: transactions.lisp Log Message: Provide checkpoint function --- /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/02/17 19:28:53 1.9 +++ /project/elephant/cvsroot/elephant/src/elephant/transactions.lisp 2007/02/18 10:58:58 1.10 @@ -147,7 +147,7 @@ ,@(remove-keywords '(:store-controller :parent :transaction :retries) keyargs)))))) -(defmacro with-batched-transaction ((batch size list &rest txn-options) &body body) +(defmacro with-batch-transaction ((batch size list &rest txn-options) &body body) "Perform a set of DB operations over a list of elements in batches of size 'size'. Pass specific transaction options after the list reference." `(loop for ,batch in (subsets ,size ,list) do

1 0

[elephant-cvs] CVS elephant/src/utils
by ieslick 17 Feb '07

17 Feb '07

Update of /project/elephant/cvsroot/elephant/src/utils In directory clnet:/tmp/cvs-serv11090/src/utils Modified Files: convenience.lisp Log Message: Bug fix in subsets function that shows up under SBCL --- /project/elephant/cvsroot/elephant/src/utils/convenience.lisp 2007/02/14 04:38:56 1.1 +++ /project/elephant/cvsroot/elephant/src/utils/convenience.lisp 2007/02/17 20:37:23 1.2 @@ -15,10 +15,10 @@ (defun subsets (size list) "Generate subsets of size n from the list; the last subset has the remaining elements if size does not represent an equal division" - (let ((subsets nil)) - (loop for elt in list + (let ((subsets (cons nil nil))) + (loop for elt in list for i from 0 do - (when (= 0 (mod i size)) + (when (and (= 0 (mod i size)) (car subsets)) (setf (car subsets) (nreverse (car subsets))) (push nil subsets)) (push elt (car subsets)))

1 0

[elephant-cvs] CVS elephant
by ieslick 17 Feb '07

17 Feb '07

Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv32211 Modified Files: TODO Log Message: Fixed unquoted macro in BDB --- /project/elephant/cvsroot/elephant/TODO 2007/02/17 12:13:18 1.52 +++ /project/elephant/cvsroot/elephant/TODO 2007/02/17 19:28:53 1.53 @@ -16,7 +16,7 @@ BDB Features/Cleanup: - Always support locks that timeout? Tradeoffs? - Perform checkpoints (prep for DCM functionality) -- Derived indices used to fail on re-connect, verify this +- Derived indices used to fail on re-connect, verify that this is fixed Lisp Support: - Win32 builds @@ -36,8 +36,6 @@ - Migration: Validate that graph structures with loop are copied properly - Migration: Improve printing and informative messages -TASKS TO GET TO FINAL RELEASE: - Test coverage: - Test for optimize storage method (just add probe-file methods to get file size?) - Multi-threading stress tests? Ensure that there are conflicts and lots of serialization @@ -46,6 +44,10 @@ - Test with UTF-16 and UTF-32 strings (construct with char-code?) - Ensure that variable length UTF-8 is automatically stored as UTF-16 +TASKS TO GET TO FINAL RELEASE: + +Fix any bugs in BETA: + Documentation: - Add document section about backend interface: - Add notes about with/ensure-transaction usage (abort & commit behavior on exit)

1 0

← Newer
1
2
3
4
5
6
7
8
...
17
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

2004

elephant-cvs February 2007