Update of /project/elephant/cvsroot/elephant/src/db-bdb
In directory clnet:/tmp/cvs-serv16382/src/db-bdb
Modified Files:
bdb-collections.lisp bdb-controller.lisp libberkeley-db.c
Log Message:
char to unsigned char fix in BDB; cleaned up modular serializer initialization in BDB and SQL backends and main protocol
--- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-collections.lisp 2007/02/02 23:51:58 1.14
+++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-collections.lisp 2007/02/04 04:34:56 1.15
@@ -151,7 +151,7 @@
(continue t))
(loop while continue
do
- (with-transaction (:store-controller sc)
+ (ensure-transaction (:store-controller sc)
(with-btree-cursor (cursor bt)
(if last-key
(cursor-set cursor last-key)
@@ -190,7 +190,7 @@
(buffer-write-oid (oid bt) key-buf)
(serialize key key-buf sc)
(serialize value value-buf sc)
- (with-transaction (:store-controller sc)
+ (ensure-transaction (:store-controller sc)
(db-put-buffered (controller-btrees sc)
key-buf value-buf)
(loop for index being the hash-value of indices
@@ -215,7 +215,7 @@
(with-buffer-streams (key-buf secondary-buf)
(buffer-write-oid (oid bt) key-buf)
(serialize key key-buf sc)
- (with-transaction (:store-controller sc)
+ (ensure-transaction (:store-controller sc)
(let ((value (get-value key bt)))
(when value
(let ((indices (indices-cache bt)))
--- /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/03 00:57:33 1.21
+++ /project/elephant/cvsroot/elephant/src/db-bdb/bdb-controller.lisp 2007/02/04 04:34:56 1.22
@@ -22,15 +22,16 @@
(declaim #-elephant-without-optimize (optimize (speed 3) (safety 1) (space 0) (debug 0)))
(defclass bdb-store-controller (store-controller)
- ((db :type (or null pointer-void) :accessor controller-db :initform '())
- (environment :type (or null pointer-void)
+ ((environment :type (or null pointer-void)
:accessor controller-environment)
- (oid-db :type (or null pointer-void) :accessor controller-oid-db)
- (oid-seq :type (or null pointer-void) :accessor controller-oid-seq)
+ (metadata :type (or null pointer-void) :accessor controller-metadata)
+ (db :type (or null pointer-void) :accessor controller-db :initform '())
(btrees :type (or null pointer-void) :accessor controller-btrees)
(indices :type (or null pointer-void) :accessor controller-indices)
(indices-assoc :type (or null pointer-void)
:accessor controller-indices-assoc)
+ (oid-db :type (or null pointer-void) :accessor controller-oid-db)
+ (oid-seq :type (or null pointer-void) :accessor controller-oid-seq)
(deadlock-pid :accessor controller-deadlock-pid :initform nil)
(deadlock-input :accessor controller-deadlock-input :initform nil))
(:documentation "Class of objects responsible for the
@@ -62,9 +63,11 @@
;;
(defmethod open-controller ((sc bdb-store-controller) &key (recover nil)
- (recover-fatal nil) (thread t) ;; (errfile nil)
+ (recover-fatal nil) (thread t)
(deadlock-detect nil))
- (let ((env (db-env-create)))
+ (let ((env (db-env-create))
+ (new-p (not (probe-file (make-pathname :defaults (second (controller-spec sc))
+ :name "%ELEPHANT")))))
(setf (controller-environment sc) env)
(db-env-set-flags env 0 :auto-commit t)
(db-env-open env (namestring (second (controller-spec sc)))
@@ -74,10 +77,25 @@
)
(db-env-set-timeout env 100000 :set-transaction-timeout t)
(db-env-set-timeout env 100000 :set-lock-timeout t)
- (let ((db (db-create env))
+ (let ((metadata (db-create env))
+ (db (db-create env))
(btrees (db-create env))
(indices (db-create env))
(indices-assoc (db-create env)))
+
+ ;; Open metadata database
+ (setf (controller-metadata sc) metadata)
+ (db-open metadata :file "%ELEPHANT" :database "%METADATA"
+ :auto-commit t :type DB-BTREE :create t :thread t)
+
+ ;; Establish database version if new
+ (when new-p (set-database-version sc))
+
+ ;; Initialize serializer so we can load proper sorting C function
+ ;; based on serializer type
+ (initialize-serializer sc)
+
+ ;; Open main class, slot-value and index databases
(setf (controller-db sc) db)
(db-open db :file "%ELEPHANT" :database "%ELEPHANTDB"
:auto-commit t :type DB-BTREE :create t :thread thread)
@@ -120,9 +138,6 @@
(setf (slot-value sc 'class-root)
(make-instance 'bdb-btree :from-oid -2 :sc sc))
-;; (when errfile
-;; (db-set-error-file (controller-db sc) errfile))
-
(when deadlock-detect
(start-deadlock-detector sc))
@@ -149,6 +164,8 @@
(setf (controller-btrees sc) nil)
(db-close (controller-db sc))
(setf (controller-db sc) nil)
+ (db-close (controller-metadata sc))
+ (setf (controller-metadata sc) nil)
(db-env-close (controller-environment sc))
(setf (controller-environment sc) nil)
nil))
@@ -160,6 +177,44 @@
:txn-nosync t))
;;
+;; Store the database version
+;;
+;; For BDB this can be in a file; different backends may require a different approach.
+
+(defmethod database-version ((sc bdb-store-controller))
+ "Elephant protocol to provide the version tag or nil if unmarked"
+ (with-buffer-streams (key val)
+ (serialize-database-version-key key)
+ (let ((buf (db-get-key-buffered (controller-metadata sc)
+ key val)))
+ (if buf (deserialize-database-version-value buf)
+ nil))))
+
+(defun set-database-version (sc)
+ "Internal use when creating new database"
+ (with-buffer-streams (key val)
+ (serialize-database-version-key key)
+ (serialize-database-version-value *elephant-code-version* val)
+ (db-put-buffered (controller-metadata sc)
+ key val)
+ *elephant-code-version*))
+
+;; (defmethod old-database-version ((sc bdb-store-controller))
+;; "A version determination for a given store
+;; controller that is independant of the serializer as the
+;; serializer is dispatched based on the code version which is a
+;; list of the form '(0 6 0)"
+;; (let ((version (elephant::controller-version-cached sc)))
+;; (if version version
+;; (let ((path (make-pathname :name "VERSION" :defaults (second (controller-spec sc)))))
+;; (if (probe-file path)
+;; (with-open-file (stream path :direction :input)
+;; (setf (elephant::controller-version-cached sc) (read stream)))
+;; (with-open-file (stream path :direction :output)
+;; (setf (elephant::controller-version-cached sc)
+;; (write *elephant-code-version* :stream stream))))))))
+
+;;
;; Automated Deadlock Support
;;
@@ -242,20 +297,3 @@
:free-space free-space)))
(values (deserialize end ctrl))))
-;; Store the serializer version. For BDB this can be in a file; different backends
-;; may require a different approach.
-(defmethod database-version ((sc bdb-store-controller))
- "A version determination for a given store
- controller that is independant of the serializer as the
- serializer is dispatched based on the code version which is a
- list of the form '(0 6 0)"
- (let ((version (elephant::controller-version-cached sc)))
- (if version version
- (let ((path (make-pathname :name "VERSION" :defaults (second (controller-spec sc)))))
- (if (probe-file path)
- (with-open-file (stream path :direction :input)
- (setf (elephant::controller-version-cached sc) (read stream)))
- (with-open-file (stream path :direction :output)
- (setf (elephant::controller-version-cached sc)
- (write *elephant-code-version* :stream stream))))))))
-
--- /project/elephant/cvsroot/elephant/src/db-bdb/libberkeley-db.c 2007/01/31 20:05:37 1.7
+++ /project/elephant/cvsroot/elephant/src/db-bdb/libberkeley-db.c 2007/02/04 04:34:56 1.8
@@ -67,94 +67,94 @@
/* should these be in network-byte order? probably not..... */
/* Pointer arithmetic utility functions */
/* should these be in network-byte order? probably not..... */
-int read_int(char *buf, int offset) {
+int read_int(unsigned char *buf, int offset) {
int i;
memcpy(&i, buf+offset, sizeof(int));
return i;
}
-int read_uint(char *buf, int offset) {
+int read_uint(unsigned char *buf, int offset) {
unsigned int ui;
memcpy(&ui, buf+offset, sizeof(unsigned int));
return ui;
}
-int32_t read_int32(char *buf, int offset) {
+int32_t read_int32(unsigned char *buf, int offset) {
int32_t i;
memcpy(&i, buf+offset, sizeof(int32_t));
return i;
}
-uint32_t read_uint32(char *buf, int offset) {
+uint32_t read_uint32(unsigned char *buf, int offset) {
uint32_t ui;
memcpy(&ui, buf+offset, sizeof(uint32_t));
return ui;
}
-int64_t read_int64(char *buf, int offset) {
+int64_t read_int64(unsigned char *buf, int offset) {
int64_t i;
memcpy(&i, buf+offset, sizeof(int64_t));
return i;
}
-uint64_t read_uint64(char *buf, int offset) {
+uint64_t read_uint64(unsigned char *buf, int offset) {
uint64_t ui;
memcpy(&ui, buf+offset, sizeof(uint64_t));
return ui;
}
-float read_float(char *buf, int offset) {
+float read_float(unsigned char *buf, int offset) {
float f;
memcpy(&f, buf+offset, sizeof(float));
return f;
}
-double read_double(char *buf, int offset) {
+double read_double(unsigned char *buf, int offset) {
double d;
memcpy(&d, buf+offset, sizeof(double));
return d;
}
/* Platform specific integer */
-void write_int(char *buf, int num, int offset) {
+void write_int(unsigned char *buf, int num, int offset) {
memcpy(buf+offset, &num, sizeof(int));
}
-void write_uint(char *buf, unsigned int num, int offset) {
+void write_uint(unsigned char *buf, unsigned int num, int offset) {
memcpy(buf+offset, &num, sizeof(unsigned int));
}
/* Well-defined integer widths */
-void write_int32(char *buf, int32_t num, int offset) {
+void write_int32(unsigned char *buf, int32_t num, int offset) {
memcpy(buf+offset, &num, sizeof(int32_t));
}
-void write_uint32(char *buf, uint32_t num, int offset) {
+void write_uint32(unsigned char *buf, uint32_t num, int offset) {
memcpy(buf+offset, &num, sizeof(uint32_t));
}
-void write_int64(char *buf, int64_t num, int offset) {
+void write_int64(unsigned char *buf, int64_t num, int offset) {
memcpy(buf+offset, &num, sizeof(int64_t));
}
-void write_uint64(char *buf, uint64_t num, int offset) {
+void write_uint64(unsigned char *buf, uint64_t num, int offset) {
memcpy(buf+offset, &num, sizeof(uint64_t));
}
-void write_float(char *buf, float num, int offset) {
+void write_float(unsigned char *buf, float num, int offset) {
memcpy(buf+offset, &num, sizeof(float));
}
-void write_double(char *buf, double num, int offset) {
+void write_double(unsigned char *buf, double num, int offset) {
memcpy(buf+offset, &num, sizeof(double));
}
-char *offset_charp(char *p, int offset) {
+unsigned char *offset_charp(unsigned char *p, int offset) {
return p + offset;
}
-void copy_buf(char *dest, int dest_offset, char *src, int src_offset,
+void copy_buf(unsigned char *dest, int dest_offset, unsigned char *src, int src_offset,
int length) {
memcpy(dest + dest_offset, src + src_offset, length);
}
@@ -177,7 +177,7 @@
return envp;
}
-char * db_strerr(int error) {
+char *db_strerr(int error) {
return db_strerror(error);
}
@@ -275,8 +275,8 @@
/* We manage our own buffers (DB_DBT_USERMEM). */
int db_get_raw(DB *db, DB_TXN *txnid,
- char *key, u_int32_t key_size,
- char *buffer, u_int32_t buffer_length,
+ unsigned char *key, u_int32_t key_size,
+ unsigned char *buffer, u_int32_t buffer_length,
u_int32_t flags, u_int32_t *result_size) {
DBT DBTKey, DBTValue;
int ret;
@@ -296,8 +296,8 @@
}
int db_put_raw(DB *db, DB_TXN *txnid,
- char *key, u_int32_t key_size,
- char *value, u_int32_t value_size,
+ unsigned char *key, u_int32_t key_size,
+ unsigned char *value, u_int32_t value_size,
u_int32_t flags) {
DBT DBTKey, DBTValue;
@@ -312,7 +312,7 @@
}
int db_del(DB *db, DB_TXN *txnid,
- char *key, u_int32_t key_size,
+ unsigned char *key, u_int32_t key_size,
u_int32_t flags) {
DBT DBTKey;
@@ -323,10 +323,10 @@
}
int db_compact(DB *db, DB_TXN *txnid,
- char *start, u_int32_t start_size,
- char *stop, u_int32_t stop_size,
+ unsigned char *start, u_int32_t start_size,
+ unsigned char *stop, u_int32_t stop_size,
u_int32_t flags,
- char *end, u_int32_t end_length,
+ unsigned char *end, u_int32_t end_length,
u_int32_t *end_size) {
DBT DBTStart, DBTStop, DBTEnd;
int errno;
@@ -380,9 +380,9 @@
}
int db_cursor_get_raw(DBC *cursor,
- char *keybuf, u_int32_t keybuf_size,
+ unsigned char *keybuf, u_int32_t keybuf_size,
u_int32_t keybuf_length,
- char *buffer, u_int32_t buffer_size,
+ unsigned char *buffer, u_int32_t buffer_size,
u_int32_t buffer_length,
u_int32_t flags, u_int32_t *ret_key_size,
u_int32_t *result_size) {
@@ -408,11 +408,11 @@
}
int db_cursor_pget_raw(DBC *cursor,
- char *keybuf, u_int32_t keybuf_size,
+ unsigned char *keybuf, u_int32_t keybuf_size,
u_int32_t keybuf_length,
- char *pkeybuf, u_int32_t pkeybuf_size,
+ unsigned char *pkeybuf, u_int32_t pkeybuf_size,
u_int32_t pkeybuf_length,
- char *buffer, u_int32_t buffer_size,
+ unsigned char *buffer, u_int32_t buffer_size,
u_int32_t buffer_length,
u_int32_t flags,
u_int32_t *ret_key_size,
@@ -446,8 +446,8 @@
}
int db_cursor_put_raw(DBC *cursor,
- char *key, u_int32_t key_size,
- char *value, u_int32_t value_size,
+ unsigned char *key, u_int32_t key_size,
+ unsigned char *value, u_int32_t value_size,
u_int32_t flags) {
DBT DBTKey, DBTValue;
@@ -465,8 +465,8 @@
/* Silently does nothing if the key/value isn't found.
Can't use auto-commit here! */
int db_del_kv(DB *db, DB_TXN *tid,
- char *key, u_int32_t key_size,
- char *value, u_int32_t value_size) {
+ unsigned char *key, u_int32_t key_size,
+ unsigned char *value, u_int32_t value_size) {
DBT DBTKey, DBTValue;
DBC *cursor;
int ret, c_ret;
@@ -495,9 +495,9 @@
/* Bulk retrieval */
int db_cursor_get_multiple_key(DBC *cursor,
- char *keybuf, u_int32_t keybuf_size,
+ unsigned char *keybuf, u_int32_t keybuf_size,
u_int32_t keybuf_length,
- char *buffer, u_int32_t buffer_size,
+ unsigned char *buffer, u_int32_t buffer_size,
u_int32_t buffer_length,
u_int32_t flags, u_int32_t *ret_key_size,
u_int32_t *result_size,
@@ -568,7 +568,7 @@
}
int db_sequence_open(DB_SEQUENCE *seq, DB_TXN *txnid,
- char *key, u_int32_t key_size, u_int32_t flags) {
+ unsigned char *key, u_int32_t key_size, u_int32_t flags) {
DBT DBTKey;
memset(&DBTKey, 0, sizeof(DBT));
DBTKey.data = key;
@@ -667,7 +667,7 @@
}
int db_env_lock_get(DB_ENV *env, u_int32_t locker,
- u_int32_t flags, char *object, u_int32_t object_size,
+ u_int32_t flags, unsigned char *object, u_int32_t object_size,
const db_lockmode_t lock_mode, DB_LOCK *lock) {
DBT DBTObject;
memset(&DBTObject, 0, sizeof(DBT));
@@ -728,8 +728,8 @@
/* Poor man's counters */
int next_counter(DB_ENV *env, DB *db, DB_TXN *parent,
- char *key, u_int32_t key_size,
- char *lockid, u_int32_t lockid_size) {
+ unsigned char *key, u_int32_t key_size,
+ unsigned char *lockid, u_int32_t lockid_size) {
DB_LOCK lock;
DBT DBTKey, DBTData;
DB_TXN *tid;
@@ -822,12 +822,14 @@
#include <math.h>
-double read_num(char *buf);
-int case_cmp(const char *a, int32_t length1, const char *b, int32_t length2);
+#define S_RESERVED 0xF0
+
+double read_num(unsigned char *buf);
+int case_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2);
int wcs_cmp(const wchar_t *a, int32_t length1, const wchar_t *b, int32_t length2);
-int lex_cmp(const char *a, int32_t length1, const char *b, int32_t length2);
-int utf16_cmp(const char *s1, int32_t length1,
- const char *s2, int32_t length2);
+int lex_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2);
+int utf16_cmp(const unsigned char *s1, int32_t length1,
+ const unsigned char *s2, int32_t length2);
#define S1_FIXNUM 1
#define S1_CHAR 2
@@ -867,9 +869,9 @@
int lisp_compare1(DB *dbp, const DBT *a, const DBT *b) {
int difference;
double ddifference;
- char *ad, *bd, at, bt;
- ad = (char*)a->data;
- bd = (char*)b->data;
+ unsigned char *ad, *bd, at, bt;
+ ad = (unsigned char *)a->data;
+ bd = (unsigned char *)b->data;
/* Compare OIDs. */
difference = read_int(ad, 0) - read_int(bd, 0);
@@ -877,7 +879,7 @@
/* Have a type tag? */
if (a->size == 4)
- if (b->size == 4)
+ if (b->size == 4)
return 0;
else
return -1;
@@ -903,6 +905,8 @@
switch (at) {
case S1_NIL: /* nil */
return 0;
+ case S_RESERVED:
+ return ad[5] < bd[5]; /* different tags */
case S1_UCS1_SYMBOL: /* 8-bit symbol */
case S1_UCS1_STRING: /* 8-bit string */
case S1_UCS1_PATHNAME: /* 8-bit pathname */
@@ -924,8 +928,8 @@
#define exp2(c) (pow(2,(c)))
#endif
-double read_num(char *buf) {
- char *limit;
+double read_num(unsigned char *buf) {
+ unsigned char *limit;
double i, result, denom;
switch (buf[0]) {
case S1_FIXNUM:
@@ -1026,6 +1030,7 @@
#define S2_STRUCT 20
#define S2_CLASS 21
#define S2_NIL 0x3F
+
#define S2_FILL_POINTER_P 0x40
#define S2_ADJUSTABLE_P 0x80
@@ -1040,16 +1045,16 @@
*****/
-double read_num2(char *buf);
+double read_num2(unsigned char *buf);
/* New serializer */
int lisp_compare2(DB *dbp, const DBT *a, const DBT *b) {
int difference;
int offset;
double ddifference;
- char *ad, *bd, at, bt;
- ad = (char*)a->data;
- bd = (char*)b->data;
+ unsigned char *ad, *bd, at, bt;
+ ad = (unsigned char *)a->data;
+ bd = (unsigned char *)b->data;
/* Compare OIDs: OIDs are limited by native integer width */
difference = read_int(ad, 0) - read_int(bd, 0);
@@ -1103,6 +1108,8 @@
switch (at) {
case S2_NIL: /* nil */
return 0;
+ case S_RESERVED:
+ return ad[5] < bd[5]; /* different tags */
case S2_UTF8_STRING: /* 8-bit string */
return case_cmp(ad+9+offset, read_int32(ad+offset, 5), bd+9+offset, read_int32(bd+offset, 5));
case S2_UTF16_STRING: /* 16-bit string */
@@ -1134,8 +1141,8 @@
}
}
-double read_num2(char *buf) {
- char *limit;
+double read_num2(unsigned char *buf) {
+ unsigned char *limit;
double i, result, denom;
switch (buf[0]) {
case S2_FIXNUM32:
@@ -1219,11 +1226,11 @@
typedef unsigned short uint16_t;
#endif
-int case_cmp(const char *a, int32_t length1, const char *b, int32_t length2) {
+int case_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2) {
int min, sizediff, diff;
sizediff = length1 - length2;
min = sizediff > 0 ? length2 : length1;
- diff = strncasecmp(a, b, min);
+ diff = strncasecmp((char *)a, (char *)b, min);
if (diff == 0) return sizediff;
return diff;
}
@@ -1238,7 +1245,7 @@
return diff;
}
-int lex_cmp(const char *a, int32_t length1, const char *b, int32_t length2) {
+int lex_cmp(const unsigned char *a, int32_t length1, const unsigned char *b, int32_t length2) {
int min, sizediff, diff;
sizediff = length1 - length2;
min = sizediff > 0 ? length2 : length1;
@@ -1258,9 +1265,9 @@
/* compare UTF-16 strings */
/* memcmp/UnicodeString style, both length-specified */
/* don't assume byte-aligned! */
-int utf16_cmp(const char *s1, int32_t length1,
- const char *s2, int32_t length2) {
- const char *start1, *start2, *limit1, *limit2;
[6 lines skipped]
