Update of /project/elephant/cvsroot/elephant/src/db-bdb In directory clnet:/tmp/cvs-serv16382/src/db-bdb
Modified Files: bdb-collections.lisp bdb-controller.lisp libberkeley-db.c Log Message: char to unsigned char fix in BDB; cleaned up modular serializer initialization in BDB and SQL backends and main protocol
--- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-collections.lisp 2007/02/02 23:51:58 1.14 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-collections.lisp 2007/02/04 04:34:56 1.15 @@ -151,7 +151,7 @@ (continue t)) (loop while continue do - (with-transaction (:store-controller sc) + (ensure-transaction (:store-controller sc) (with-btree-cursor (cursor bt) (if last-key (cursor-set cursor last-key) @@ -190,7 +190,7 @@ (buffer-write-oid (oid bt) key-buf) (serialize key key-buf sc) (serialize value value-buf sc) - (with-transaction (:store-controller sc) + (ensure-transaction (:store-controller sc) (db-put-buffered (controller-btrees sc) key-buf value-buf) (loop for index being the hash-value of indices @@ -215,7 +215,7 @@ (with-buffer-streams (key-buf secondary-buf) (buffer-write-oid (oid bt) key-buf) (serialize key key-buf sc) - (with-transaction (:store-controller sc) + (ensure-transaction (:store-controller sc) (let ((value (get-value key bt))) (when value (let ((indices (indices-cache bt))) --- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/03 00:57:33 1.21 +++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/04 04:34:56 1.22 @@ -22,15 +22,16 @@ (declaim #-elephant-without-optimize (optimize (speed 3) (safety 1) (space 0) (debug 0)))
(defclass bdb-store-controller (store-controller) - ((db :type (or null pointer-void) :accessor controller-db :initform '()) - (environment :type (or null pointer-void) + ((environment :type (or null pointer-void) :accessor controller-environment) - (oid-db :type (or null pointer-void) :accessor controller-oid-db) - (oid-seq :type (or null pointer-void) :accessor controller-oid-seq) + (metadata :type (or null pointer-void) :accessor controller-metadata) + (db :type (or null pointer-void) :accessor controller-db :initform '()) (btrees :type (or null pointer-void) :accessor controller-btrees) (indices :type (or null pointer-void) :accessor controller-indices) (indices-assoc :type (or null pointer-void) :accessor controller-indices-assoc) + (oid-db :type (or null pointer-void) :accessor controller-oid-db) + (oid-seq :type (or null pointer-void) :accessor controller-oid-seq) (deadlock-pid :accessor controller-deadlock-pid :initform nil) (deadlock-input :accessor controller-deadlock-input :initform nil)) (:documentation "Class of objects responsible for the @@ -62,9 +63,11 @@ ;;
(defmethod open-controller ((sc bdb-store-controller) &key (recover nil) - (recover-fatal nil) (thread t) ;; (errfile nil) + (recover-fatal nil) (thread t) (deadlock-detect nil)) - (let ((env (db-env-create))) + (let ((env (db-env-create)) + (new-p (not (probe-file (make-pathname :defaults (second (controller-spec sc)) + :name "%ELEPHANT"))))) (setf (controller-environment sc) env) (db-env-set-flags env 0 :auto-commit t) (db-env-open env (namestring (second (controller-spec sc))) @@ -74,10 +77,25 @@ ) (db-env-set-timeout env 100000 :set-transaction-timeout t) (db-env-set-timeout env 100000 :set-lock-timeout t) - (let ((db (db-create env)) + (let ((metadata (db-create env)) + (db (db-create env)) (btrees (db-create env)) (indices (db-create env)) (indices-assoc (db-create env))) + + ;; Open metadata database + (setf (controller-metadata sc) metadata) + (db-open metadata :file "%ELEPHANT" :database "%METADATA" + :auto-commit t :type DB-BTREE :create t :thread t) + + ;; Establish database version if new + (when new-p (set-database-version sc)) + + ;; Initialize serializer so we can load proper sorting C function + ;; based on serializer type + (initialize-serializer sc) + + ;; Open main class, slot-value and index databases (setf (controller-db sc) db) (db-open db :file "%ELEPHANT" :database "%ELEPHANTDB" :auto-commit t :type DB-BTREE :create t :thread thread) @@ -120,9 +138,6 @@ (setf (slot-value sc 'class-root) (make-instance 'bdb-btree :from-oid -2 :sc sc))
-;; (when errfile -;; (db-set-error-file (controller-db sc) errfile)) - (when deadlock-detect (start-deadlock-detector sc))
@@ -149,6 +164,8 @@ (setf (controller-btrees sc) nil) (db-close (controller-db sc)) (setf (controller-db sc) nil) + (db-close (controller-metadata sc)) + (setf (controller-metadata sc) nil) (db-env-close (controller-environment sc)) (setf (controller-environment sc) nil) nil)) @@ -160,6 +177,44 @@ :txn-nosync t))
;; +;; Store the database version +;; +;; For BDB this can be in a file; different backends may require a different approach. + +(defmethod database-version ((sc bdb-store-controller)) + "Elephant protocol to provide the version tag or nil if unmarked" + (with-buffer-streams (key val) + (serialize-database-version-key key) + (let ((buf (db-get-key-buffered (controller-metadata sc) + key val))) + (if buf (deserialize-database-version-value buf) + nil)))) + +(defun set-database-version (sc) + "Internal use when creating new database" + (with-buffer-streams (key val) + (serialize-database-version-key key) + (serialize-database-version-value *elephant-code-version* val) + (db-put-buffered (controller-metadata sc) + key val) + *elephant-code-version*)) + +;; (defmethod old-database-version ((sc bdb-store-controller)) +;; "A version determination for a given store +;; controller that is independant of the serializer as the +;; serializer is dispatched based on the code version which is a +;; list of the form '(0 6 0)" +;; (let ((version (elephant::controller-version-cached sc))) +;; (if version version +;; (let ((path (make-pathname :name "VERSION" :defaults (second (controller-spec sc))))) +;; (if (probe-file path) +;; (with-open-file (stream path :direction :input) +;; (setf (elephant::controller-version-cached sc) (read stream))) +;; (with-open-file (stream path :direction :output) +;; (setf (elephant::controller-version-cached sc) +;; (write *elephant-code-version* :stream stream)))))))) + +;; ;; Automated Deadlock Support ;;
@@ -242,20 +297,3 @@ :free-space free-space))) (values (deserialize end ctrl))))
-;; Store the serializer version. For BDB this can be in a file; different backends -;; may require a different approach. -(defmethod database-version ((sc bdb-store-controller)) - "A version determination for a given store - controller that is independant of the serializer as the - serializer is dispatched based on the code version which is a - list of the form '(0 6 0)" - (let ((version (elephant::controller-version-cached sc))) - (if version version - (let ((path (make-pathname :name "VERSION" :defaults (second (controller-spec sc))))) - (if (probe-file path) - (with-open-file (stream path :direction :input) - (setf (elephant::controller-version-cached sc) (read stream))) - (with-open-file (stream path :direction :output) - (setf (elephant::controller-version-cached sc) - (write *elephant-code-version* :stream stream)))))))) - --- /project/elephant/cvsroot/elephant/src/db-bdb/libberkeley-db.c 2007/01/31 20:05:37 1.7 +++ /project/elephant/cvsroot/elephant/src/db-bdb/libberkeley-db.c 2007/02/04 04:34:56 1.8 @@ -67,94 +67,94 @@ /* should these be in network-byte order? probably not..... */ /* Pointer arithmetic utility functions */ /* should these be in network-byte order? probably not..... */ -int read_int(char *buf, int offset) { +int read_int(unsigned char *buf, int offset) { int i; memcpy(&i, buf+offset, sizeof(int)); return i; }
-int read_uint(char *buf, int offset) { +int read_uint(unsigned char *buf, int offset) { unsigned int ui; memcpy(&ui, buf+offset, sizeof(unsigned int)); return ui; }
-int32_t read_int32(char *buf, int offset) { +int32_t read_int32(unsigned char *buf, int offset) { int32_t i; memcpy(&i, buf+offset, sizeof(int32_t)); return i; }
-uint32_t read_uint32(char *buf, int offset) { +uint32_t read_uint32(unsigned char *buf, int offset) { uint32_t ui; memcpy(&ui, buf+offset, sizeof(uint32_t)); return ui; }
-int64_t read_int64(char *buf, int offset) { +int64_t read_int64(unsigned char *buf, int offset) { int64_t i; memcpy(&i, buf+offset, sizeof(int64_t)); return i; }
-uint64_t read_uint64(char *buf, int offset) { +uint64_t read_uint64(unsigned char *buf, int offset) { uint64_t ui; memcpy(&ui, buf+offset, sizeof(uint64_t)); return ui; }
-float read_float(char *buf, int offset) { +float read_float(unsigned char *buf, int offset) { float f; memcpy(&f, buf+offset, sizeof(float)); return f; }
-double read_double(char *buf, int offset) { +double read_double(unsigned char *buf, int offset) { double d; memcpy(&d, buf+offset, sizeof(double)); return d; }
/* Platform specific integer */ -void write_int(char *buf, int num, int offset) { +void write_int(unsigned char *buf, int num, int offset) { memcpy(buf+offset, &num, sizeof(int)); }
-void write_uint(char *buf, unsigned int num, int offset) { +void write_uint(unsigned char *buf, unsigned int num, int offset) { memcpy(buf+offset, &num, sizeof(unsigned int)); }
/* Well-defined integer widths */ -void write_int32(char *buf, int32_t num, int offset) { +void write_int32(unsigned char *buf, int32_t num, int offset) { memcpy(buf+offset, &num, sizeof(int32_t)); }
-void write_uint32(char *buf, uint32_t num, int offset) { +void write_uint32(unsigned char *buf, uint32_t num, int offset) { memcpy(buf+offset, &num, sizeof(uint32_t)); }
-void write_int64(char *buf, int64_t num, int offset) { +void write_int64(unsigned char *buf, int64_t num, int offset) { memcpy(buf+offset, &num, sizeof(int64_t)); }
-void write_uint64(char *buf, uint64_t num, int offset) { +void write_uint64(unsigned char *buf, uint64_t num, int offset) { memcpy(buf+offset, &num, sizeof(uint64_t)); }
-void write_float(char *buf, float num, int offset) { +void write_float(unsigned char *buf, float num, int offset) { memcpy(buf+offset, &num, sizeof(float)); }
-void write_double(char *buf, double num, int offset) { +void write_double(unsigned char *buf, double num, int offset) { memcpy(buf+offset, &num, sizeof(double)); }
-char *offset_charp(char *p, int offset) { +unsigned char *offset_charp(unsigned char *p, int offset) { return p + offset; }
-void copy_buf(char *dest, int dest_offset, char *src, int src_offset, +void copy_buf(unsigned char *dest, int dest_offset, unsigned char *src, int src_offset, int length) { memcpy(dest + dest_offset, src + src_offset, length); } @@ -177,7 +177,7 @@ return envp; }
-char * db_strerr(int error) { +char *db_strerr(int error) { return db_strerror(error); }
@@ -275,8 +275,8 @@ /* We manage our own buffers (DB_DBT_USERMEM). */
int db_get_raw(DB *db, DB_TXN *txnid, - char *key, u_int32_t key_size, - char *buffer, u_int32_t buffer_length, + unsigned char *key, u_int32_t key_size, + unsigned char *buffer, u_int32_t buffer_length, u_int32_t flags, u_int32_t *result_size) { DBT DBTKey, DBTValue; int ret; @@ -296,8 +296,8 @@ }
int db_put_raw(DB *db, DB_TXN *txnid, - char *key, u_int32_t key_size, - char *value, u_int32_t value_size, + unsigned char *key, u_int32_t key_size, + unsigned char *value, u_int32_t value_size, u_int32_t flags) { DBT DBTKey, DBTValue;
@@ -312,7 +312,7 @@ }
int db_del(DB *db, DB_TXN *txnid, - char *key, u_int32_t key_size, + unsigned char *key, u_int32_t key_size, u_int32_t flags) { DBT DBTKey;
@@ -323,10 +323,10 @@ }
int db_compact(DB *db, DB_TXN *txnid, - char *start, u_int32_t start_size, - char *stop, u_int32_t stop_size, + unsigned char *start, u_int32_t start_size, + unsigned char *stop, u_int32_t stop_size, u_int32_t flags, - char *end, u_int32_t end_length, + unsigned char *end, u_int32_t end_length, u_int32_t *end_size) { DBT DBTStart, DBTStop, DBTEnd; int errno; @@ -380,9 +380,9 @@ }
int db_cursor_get_raw(DBC *cursor, - char *keybuf, u_int32_t keybuf_size, + unsigned char *keybuf, u_int32_t keybuf_size, u_int32_t keybuf_length, - char *buffer, u_int32_t buffer_size, + unsigned char *buffer, u_int32_t buffer_size, u_int32_t buffer_length, u_int32_t flags, u_int32_t *ret_key_size, u_int32_t *result_size) { @@ -408,11 +408,11 @@ }
int db_cursor_pget_raw(DBC *cursor, - char *keybuf, u_int32_t keybuf_size, + unsigned char *keybuf, u_int32_t keybuf_size, u_int32_t keybuf_length, - char *pkeybuf, u_int32_t pkeybuf_size, + unsigned char *pkeybuf, u_int32_t pkeybuf_size, u_int32_t pkeybuf_length, - char *buffer, u_int32_t buffer_size, + unsigned char *buffer, u_int32_t buffer_size, u_int32_t buffer_length, u_int32_t flags, u_int32_t *ret_key_size, @@ -446,8 +446,8 @@ }
int db_cursor_put_raw(DBC *cursor, - char *key, u_int32_t key_size, - char *value, u_int32_t value_size, + unsigned char *key, u_int32_t key_size, + unsigned char *value, u_int32_t value_size, u_int32_t flags) { DBT DBTKey, DBTValue;
@@ -465,8 +465,8 @@ /* Silently does nothing if the key/value isn't found. Can't use auto-commit here! */ int db_del_kv(DB *db, DB_TXN *tid, - char *key, u_int32_t key_size, - char *value, u_int32_t value_size) { + unsigned char *key, u_int32_t key_size, + unsigned char *value, u_int32_t value_size) { DBT DBTKey, DBTValue; DBC *cursor; int ret, c_ret; @@ -495,9 +495,9 @@ /* Bulk retrieval */
int db_cursor_get_multiple_key(DBC *cursor, - char *keybuf, u_int32_t keybuf_size, + unsigned char *keybuf, u_int32_t keybuf_size, u_int32_t keybuf_length, - char *buffer, u_int32_t buffer_size, + unsigned char *buffer, u_int32_t buffer_size, u_int32_t buffer_length, u_int32_t flags, u_int32_t *ret_key_size, u_int32_t *result_size, @@ -568,7 +568,7 @@ }
int db_sequence_open(DB_SEQUENCE *seq, DB_TXN *txnid, - char *key, u_int32_t key_size, u_int32_t flags) { + unsigned char *key, u_int32_t key_size, u_int32_t flags) { DBT DBTKey; memset(&DBTKey, 0, sizeof(DBT)); DBTKey.data = key; @@ -667,7 +667,7 @@ }
int db_env_lock_get(DB_ENV *env, u_int32_t locker, - u_int32_t flags, char *object, u_int32_t object_size, + u_int32_t flags, unsigned char *object, u_int32_t object_size, const db_lockmode_t lock_mode, DB_LOCK *lock) { DBT DBTObject; memset(&DBTObject, 0, sizeof(DBT)); @@ -728,8 +728,8 @@ /* Poor man's counters */
int next_counter(DB_ENV *env, DB *db, DB_TXN *parent, - char *key, u_int32_t key_size, - char *lockid, u_int32_t lockid_size) { + unsigned char *key, u_int32_t key_size, + unsigned char *lockid, u_int32_t lockid_size) { DB_LOCK lock; DBT DBTKey, DBTData; DB_TXN *tid; @@ -822,12 +822,14 @@
#include <math.h>
-double read_num(char *buf); -int case_cmp(const char *a, int32_t length1, const char *b, int32_t length2); +#define S_RESERVED 0xF0 + +double read_num(unsigned char *buf); +int case_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2); int wcs_cmp(const wchar_t *a, int32_t length1, const wchar_t *b, int32_t length2); -int lex_cmp(const char *a, int32_t length1, const char *b, int32_t length2); -int utf16_cmp(const char *s1, int32_t length1, - const char *s2, int32_t length2); +int lex_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2); +int utf16_cmp(const unsigned char *s1, int32_t length1, + const unsigned char *s2, int32_t length2);
#define S1_FIXNUM 1 #define S1_CHAR 2 @@ -867,9 +869,9 @@ int lisp_compare1(DB *dbp, const DBT *a, const DBT *b) { int difference; double ddifference; - char *ad, *bd, at, bt; - ad = (char*)a->data; - bd = (char*)b->data; + unsigned char *ad, *bd, at, bt; + ad = (unsigned char *)a->data; + bd = (unsigned char *)b->data;
/* Compare OIDs. */ difference = read_int(ad, 0) - read_int(bd, 0); @@ -877,7 +879,7 @@
/* Have a type tag? */ if (a->size == 4) - if (b->size == 4) + if (b->size == 4) return 0; else return -1; @@ -903,6 +905,8 @@ switch (at) { case S1_NIL: /* nil */ return 0; + case S_RESERVED: + return ad[5] < bd[5]; /* different tags */ case S1_UCS1_SYMBOL: /* 8-bit symbol */ case S1_UCS1_STRING: /* 8-bit string */ case S1_UCS1_PATHNAME: /* 8-bit pathname */ @@ -924,8 +928,8 @@ #define exp2(c) (pow(2,(c))) #endif
-double read_num(char *buf) { - char *limit; +double read_num(unsigned char *buf) { + unsigned char *limit; double i, result, denom; switch (buf[0]) { case S1_FIXNUM: @@ -1026,6 +1030,7 @@ #define S2_STRUCT 20 #define S2_CLASS 21 #define S2_NIL 0x3F + #define S2_FILL_POINTER_P 0x40 #define S2_ADJUSTABLE_P 0x80
@@ -1040,16 +1045,16 @@
*****/
-double read_num2(char *buf); +double read_num2(unsigned char *buf);
/* New serializer */ int lisp_compare2(DB *dbp, const DBT *a, const DBT *b) { int difference; int offset; double ddifference; - char *ad, *bd, at, bt; - ad = (char*)a->data; - bd = (char*)b->data; + unsigned char *ad, *bd, at, bt; + ad = (unsigned char *)a->data; + bd = (unsigned char *)b->data;
/* Compare OIDs: OIDs are limited by native integer width */ difference = read_int(ad, 0) - read_int(bd, 0); @@ -1103,6 +1108,8 @@ switch (at) { case S2_NIL: /* nil */ return 0; + case S_RESERVED: + return ad[5] < bd[5]; /* different tags */ case S2_UTF8_STRING: /* 8-bit string */ return case_cmp(ad+9+offset, read_int32(ad+offset, 5), bd+9+offset, read_int32(bd+offset, 5)); case S2_UTF16_STRING: /* 16-bit string */ @@ -1134,8 +1141,8 @@ } }
-double read_num2(char *buf) { - char *limit; +double read_num2(unsigned char *buf) { + unsigned char *limit; double i, result, denom; switch (buf[0]) { case S2_FIXNUM32: @@ -1219,11 +1226,11 @@ typedef unsigned short uint16_t; #endif
-int case_cmp(const char *a, int32_t length1, const char *b, int32_t length2) { +int case_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2) { int min, sizediff, diff; sizediff = length1 - length2; min = sizediff > 0 ? length2 : length1; - diff = strncasecmp(a, b, min); + diff = strncasecmp((char *)a, (char *)b, min); if (diff == 0) return sizediff; return diff; } @@ -1238,7 +1245,7 @@ return diff; }
-int lex_cmp(const char *a, int32_t length1, const char *b, int32_t length2) { +int lex_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2) { int min, sizediff, diff; sizediff = length1 - length2; min = sizediff > 0 ? length2 : length1; @@ -1258,9 +1265,9 @@ /* compare UTF-16 strings */ /* memcmp/UnicodeString style, both length-specified */ /* don't assume byte-aligned! */ -int utf16_cmp(const char *s1, int32_t length1, - const char *s2, int32_t length2) { - const char *start1, *start2, *limit1, *limit2;
[6 lines skipped]