Update of /project/elephant/cvsroot/elephant In directory clnet:/tmp/cvs-serv5389

Modified Files: TODO config.sexp ele-bdb.asd elephant.asd Log Message: Bug fix in drop instances; preliminary cygwin build in asd fils

--- /project/elephant/cvsroot/elephant/TODO 2007/02/21 06:29:31 1.57 +++ /project/elephant/cvsroot/elephant/TODO 2007/02/22 20:19:57 1.58 @@ -11,26 +11,26 @@ Migration: - Validate SQL migration 0.6.0->0.6.1 (Robert)

-Lisp Support: -- Win32 builds - - Windows support for asdf-based library builds? Include 32-bit dll in release? -- 64-bit lisp verification -- Validate OpenMCL 1.1 on Mac OS X -- Validate Lispworks -- Verify db_deadlock for other lisps (launch and kill background program I/F) - Stability and Performance: - Review and address all NOTE comments in the code - Review SBCL string serialization performance -- Improve SQL base-64 serializer performance? - Migration: Validate that migrate can use either O(c) or O(n/c) where c << n memory for large DBs - Migration: Improve support for nested persistent objects inside standard objects, arrays, etc? - Migration: Improve scaling properties of migration so all objects do not need to be resident in memory? - Migration: Validate that graph structures with loop are copied properly - Migration: Improve printing and informative messages

+Lisp Support: +- Win32 builds + - Windows support for asdf-based library builds? Include 32-bit dll in release? +- Validate Lispworks +- Validate OpenMCL pre-1.1 on Mac OS X +- Validate OpenMCL 1.1 and/or 64-bit on Mac OS X? +- Verify db_deadlock for other lisps (launch and kill background program I/F) +- 64-bit lisp verification + Test coverage: -- Make tests idempotent, clean up interface to tests +- Clean up interface to tests - Test for optimize storage method (just add probe-file methods to get file size?) - Multi-threading stress tests? Ensure that there are conflicts and lots of serialization happening concurrently to make sure that multi-threading is in good shape (Henrik's code) @@ -41,16 +41,19 @@

TASKS TO GET TO FINAL RELEASE:

-Fix any bugs in BETA: +Fix any bugs found in BETA

Documentation: - License and copyright file headers -- Add document section about backend interface: -- Add notes about with/ensure-transaction usage (abort & commit behavior on exit) -- Add notes about optimize-storage -- Add notes about deadlock-detect -- Add notes about checkpoint (null in SQL?) -- More notes about transaction performance +- Add document section about backend interface & developer decisions +- Performance and design issues + - More notes about transaction performance + - Serious discussion of threading implications + - Add notes about with/ensure-transaction usage (abort & commit behavior on exit) + - Add notes about optimize-storage + - Add notes about deadlock-detect + - Add notes about checkpoint (null in SQL?) +- Upgrade, migration and other system level issues

0.6.1 - Features COMPLETED to date ---------------------------------- @@ -176,6 +179,7 @@ - Add lazy deserialize to map functions?

Performance: +- Improve SQL base-64 serializer performance? - Implement unicode performance hacks for various lisps; validate UTF8 works everywhere - Metering and understanding locking issues. Large transactions seem to use a lot of locks. In general understanding how to use Berkeley DB --- /project/elephant/cvsroot/elephant/config.sexp 2007/02/14 04:36:08 1.6 +++ /project/elephant/cvsroot/elephant/config.sexp 2007/02/22 20:19:57 1.7 @@ -3,7 +3,8 @@ (:berkeley-db-lib . "/opt/local/BerkeleyDB.4.5/lib/libDB-4.5.dylib") (:berkeley-db-deadlock . "/opt/local/bin/db45_deadlock") (:pthread-lib . nil) - (:clsql-lib . nil)) + (:clsql-lib . nil) + (:compiler . :gcc))

;; Berkeley 4.5 is required, each system will have different settings for ;; these directories, use this as an indication of what each key means @@ -13,4 +14,9 @@ ;; nil means that the library in question is not loaded ;; ;; NOTE: The latest SBCL (0.9.17+) on linux no longer needs the pthread library, -;; it is statically linked against it now with the new thread support \ No newline at end of file +;; it is statically linked against it now with the new thread support +;; +;; :compiler options are +;; :gcc (default: for unix platforms with /usr/bin/gcc) +;; :cygwin (for windows platforms with cygwin/gcc) +;; :msvc (unsupported) \ No newline at end of file --- /project/elephant/cvsroot/elephant/ele-bdb.asd 2007/01/31 20:05:37 1.16 +++ /project/elephant/cvsroot/elephant/ele-bdb.asd 2007/02/22 20:19:57 1.17 @@ -29,13 +29,18 @@

(defclass bdb-c-source (elephant-c-source) ())

-(defmethod compiler-options ((compiler (eql :gcc)) (c bdb-c-source) &key &allow-other-keys) - (let* ((include (make-pathname :directory (get-config-option :berkeley-db-include-dir c))) - (lib (make-pathname :directory (get-config-option :berkeley-db-lib-dir c)))) +(defmethod compiler-options (compiler (c bdb-c-source) &key &allow-other-keys) + (let ((include (make-pathname :directory (get-config-option :berkeley-db-include-dir c))) + (lib (make-pathname :directory (get-config-option :berkeley-db-lib-dir c)))) (append (list (format nil "-L~A" lib) (format nil "-I~A" include)) (call-next-method) (list "-ldb"))))

+;;Cygwin script: +;;gcc -mno-cygwin -mwindows -c -Wall -std=c99 -L/c/DB/Berkeley\ DB\ 4.4.20/lib/ -I/c/DB/Berkeley\ DB\ 4.4.20/include/ libsleepycat.c +;;dlltool -z libsleepycat.def --export-all-symbols -e exports.o -l libsleepycat.lib libsleepycat.o +;;gcc -shared -mno-cygwin -mwindows -L/c/DB/Berkeley\ DB\ 4.4.20/bin/ -llibdb44 libsleepycat.o exports.o -o libsleepycat.dll + (defmethod foreign-libraries-to-load-first ((c bdb-c-source)) (remove-if #'(lambda (x) (null (car x))) (list --- /project/elephant/cvsroot/elephant/elephant.asd 2007/02/20 19:12:57 1.32 +++ /project/elephant/cvsroot/elephant/elephant.asd 2007/02/22 20:19:57 1.33 @@ -28,22 +28,12 @@ ;; Simple lisp/asdf-based make utility for elephant c files ;;

-(defvar *c-compilers* - '((:gcc . "/usr/bin/gcc") - (:msvc . "")) - "Associate compilers with platforms for compiling libmemutil/libsleepycat") - -(defvar *compiler* - #-(or win32 windows) :gcc - #+(or win32 windows) :msvc) - (defgeneric compiler-options (compiler c-source-file &key input-file output-file) (:documentation "Returns a list of options to pass to <compiler>"))

(defgeneric foreign-libraries-to-load-first (c-source-file) (:documentation "Provides an alist of foreign-libraries to load and the modules to load them into. Similar to (input-files load-op), but much more specific"))

- (defun uffi-funcall (fn &rest args) "Simplify uffi funcall, first ensure uffi is loaded" (unless (find-package :uffi) @@ -54,8 +44,6 @@ ;; User parameters (bdb root and pthread, if necessary) ;;

-(defparameter *elephant-user-config* nil) - (defun get-config-option (option component) (let ((filespec (make-pathname :defaults (asdf:component-pathname (asdf:component-system component)) :name "my-config" @@ -67,6 +55,27 @@ (cdr (assoc option (read config))))))

;; +;; Supported C compilers +;; + + +(defvar *c-compilers* + '((:gcc . "/usr/bin/gcc") + (:cygwin . "c:\cygwin\usr\bin\gcc") + (:msvc . "")) + "Associate compilers with platforms for compiling libmemutil/libsleepycat") + +(defun c-compiler (comp) + (get-config-option :compiler comp)) + +(defun c-compiler-path (comp) + (let* ((compiler (get-config-option :compiler comp)) + (entry (assoc compiler *c-compilers*))) + (if entry + (cdr entry) + (error "Cannot find compiler path for config.sexp :compiler option: ~A" compiler)))) + +;; ;; Basic utilities for elephant c files ;;

@@ -85,63 +94,89 @@ "Run the appropriate compiler for this platform on the source, getting the specific options from 'compiler-options method. Default options can be overridden or augmented by subclass methods" + #+windows + (progn + (let ((pathname (component-pathname c))) + (unless (zerop (run-shell-command + (format nil "~A ~{~A ~}" + (c-compiler-path c) + (compiler-options (c-compiler c) c + :input-file (namestring pathname) + :output-file nil + :library nil)))) + (error 'operation-error :component c :operation o)) + (unless (zerop (run-shell-command + (format nil "dlltool -z ~A --export-all-symbols -e exports.o -l ~A ~A" + (namestring (make-pathname :type "def" :defaults pathname)) + (namestring (make-pathname :type "lib" :defaults pathname)) + (namestring (make-pathname :type "o" :defaults pathname))))) + (error 'operation-error :component c :operation o)) + (unless (zerop (run-shell-command + (format nil "~A ~{~A ~} -I~A -L~A -l~A" + (c-compiler-path c) + (compiler-options (c-compiler c) c + :input-files + (list (namestring + (make-pathname :type "o" :defaults pathname)) + "exports.o") + :output-file (first (output-files o c)) + :library t)))) + (error 'operation-error :component c :operation o)))) + #-windows (unless (zerop (run-shell-command "~A ~{~A ~}" - (cdr (assoc *compiler* *c-compilers*)) - (compiler-options - *compiler* - c - :input-file (namestring (component-pathname c)) - :output-file (namestring (first (output-files o c)))))) + (c-compiler-path c) + (compiler-options (c-compiler c) c + :input-file (namestring (component-pathname c)) + :output-file (namestring (first (output-files o c)))))) (error 'operation-error :component c :operation o)))

+ +gcc -mno-cygwin -mwindows -c -Wall -std=c99 -L/c/DB/Berkeley\ DB\ 4.4.20/lib/ -I/c/DB/Berkeley\ DB\ 4.4.20/include/ libsleepycat.c +dlltool -z libsleepycat.def --export-all-symbols -e exports.o -l libsleepycat.lib libsleepycat.o +gcc -shared -mno-cygwin -mwindows -L/c/DB/Berkeley\ DB\ 4.4.20/bin/ -llibdb44 libsleepycat.o exports.o -o libsleepycat.dll + + (defmethod operation-done-p ((o compile-op) (c elephant-c-source)) "Is the first generated library more recent than the source file?" - (let ((lib (first (output-files o c)))) + (let ((lib (first (output-files o c))) (and (probe-file (component-pathname c)) (probe-file lib) - (> (file-write-date lib) (file-write-date (component-pathname c)))))) + (> (file-write-date lib) (file-write-date (component-pathname c)))))))

-(defmethod compiler-options ((compiler (eql :gcc)) (c elephant-c-source) &key input-file output-file) +(defmethod compiler-options ((compiler (eql :gcc)) (c elephant-c-source) &key input-file output-file &allow-other-keys) "Default compile and link options to create a library; no -L or -I options included; math lib as default" (unless (and input-file output-file) (error "Must specify both input and output files")) - (list + (list #-(or darwin macosx darwin-host) "-shared" #+(or darwin macosx darwin-host) "-bundle" #+(and X86-64 (or macosx darwin darwin-host)) "-arch x86_64" #+(and X86-64 linux) "-march=x86-64" - "-Wall" "-fPIC" + "-Wall" "-O3" - "-o" output-file input-file + "-o" output-file "-lm"))

-;;(defmethod compiler-options ((compiler (eql :gcc-cygwin)) (c elephant-c-source) &key input-file output-file) -;; (unless (and input-file output-file) -;; (error "Must specify both input and output files")) -;; (list -;; "-shared" -;; "-mno-cygwin" -;; "-mwindows" -;; "-std=c99" -;; input-file -;; "-o" output-file -;; "--export-symbols" -;; (namestring (make-pathname :defaults output-file :type "def")))) +(defmethod compiler-options ((compiler (eql :cygwin)) (c elephant-c-source) &key input-file output-file library &allow-other-keys) + (unless input-file + (error "Must specify both input and output files")) + `(,@(when library (list "-shared")) + "-mno-cygwin" + "-mwindows" + "-Wall" + ,@(unless library (list "-c -std=c99")) + "-std=c99" + ,@(when (symbolp input-file) (list input-file) input-file) + ,@(when output-file (list "-o" output-file))))

+;;Cygwin script: ;;gcc -mno-cygwin -mwindows -std=c99 -c libmemutil.c ;;dlltool -z libmeutil.def --export-all-symbols -e exports.o -l libmemutil.lib libmemutil.o ;;gcc -shared -mno-cygwin -mwindows libmemutil.o exports.o -o libmemutil.dll

-;;gcc -shared -mno-cygwin -mwindows libmemutil.o exports.o -o libmemutil.dll - -;;And this is the script for libsleepycat.dll: -;; -;;gcc -mno-cygwin -mwindows -c -Wall -std=c99 -L/c/DB/Berkeley\ DB\ 4.4.20/lib/ -I/c/DB/Berkeley\ DB\ 4.4.20/include/ libsleepycat.c -;;dlltool -z libsleepycat.def --export-all-symbols -e exports.o -l libsleepycat.lib libsleepycat.o -;;gcc -shared -mno-cygwin -mwindows -L/c/DB/Berkeley\ DB\ 4.4.20/bin/ -llibdb44 libsleepycat.o exports.o -o libsleepycat.dll

(defmethod compiler-options ((compiler (eql :msvc)) (c elephant-c-source) &key input-file output-file) (error "MSVC compiler option not supported yet"))