Author: lgiessmann
Date: Tue Dec 14 16:07:50 2010
New Revision: 362
Log:
TM-SPARQL: added some functions that separate a single filter-statement, handle bracketing, and handle unsupported functions
Modified:
trunk/src/TM-SPARQL/sparql_filter.lisp
trunk/src/TM-SPARQL/sparql_parser.lisp
trunk/src/base-tools/base-tools.lisp
trunk/src/unit_tests/sparql_test.lisp
Modified: trunk/src/TM-SPARQL/sparql_filter.lisp
==============================================================================
--- trunk/src/TM-SPARQL/sparql_filter.lisp (original)
+++ trunk/src/TM-SPARQL/sparql_filter.lisp Tue Dec 14 16:07:50 2010
@@ -9,14 +9,42 @@
(in-package :TM-SPARQL)
-(defun parse-filter (query-string query-object)
- "A helper functions that returns a filter and the next-query string
- in the form (:next-query string :filter object)."
- (declare (String query-string)
- (SPARQL-Query query-object))
+
+(defparameter *supported-functions*
+ (list "BOUND" "isLITERAL" "STR" "DATATYPE" "REGEX")
+ "Contains all supported SPARQL-functions")
+
+
+(defparameter *supported-operators*
+ (list "!" "||" "&&" "=" "!=" "<" "<=" ">" ">=" "+" "-" "*" "/")
+ "Contains all supported operators, note some unary operators
+ are handled as functions, e.g. + and -")
+
+
+(defun make-sparql-parser-condition(rest-of-query entire-query expected)
+ "Creates a spqrql-parser-error object."
+ (declare (String rest-of-query entire-query expected))
+ (let ((message
+ (format nil "The query:~%\"~a\"~%~%has a bad token at position ~a => ~a.~%Expected: ~a"
+ entire-query (- (length entire-query)
+ (length rest-of-query))
+ (subseq entire-query (- (length entire-query)
+ (length rest-of-query)))
+ expected)))
+ (make-condition 'sparql-parser-error :message message)))
+
+
+(defgeneric parse-filter (construct query-string)
+ (:documentation "A helper functions that returns a filter and the next-query
+ string in the form (:next-query string :filter object).")
+ (:method ((construct SPARQL-Query) (query-string String))
+ (let* ((result-set-boundings (set-boundings construct query-string))
+ (filter-string (getf result-set-boundings :filter-string))
+ (next-query (getf result-set-boundings :next-query))
+ ))))
;;TODO: implement
- ;; *replace () by (progn )
- ;; *replace ', """, ''' by "
+ ;; **replace () by (progn )
+ ;; **replace ', """, ''' by '''
;; *replace !x by (not x)
;; *replace +x by (1+ x)
;; *replace -x by (1- x)
@@ -25,7 +53,147 @@
;; *replace function(x), function(x, y), function(x, y, z)
;; by filter-function(x), (filter-function(x, y), filter-function(x, y, z)
;; *create and store this filter object
- )
+
+
+(defgeneric set-boundings (construct query-string)
+ (:documentation "Returns a list of the form (:next-query <string>
+ :filter-string <string>). next-query is a string containing
+ the query after the filter and filter is a string
+ containing the actual filter. Additionally all free
+ '(' are transformed into '(progn' and all ', ''', \"\"\"
+ are transformed into \".")
+ (:method ((construct SPARQL-Query) (query-string String))
+ (let ((filter-string "")
+ (open-brackets 0)
+ (result nil))
+ (dotimes (idx (length query-string))
+ (let ((current-char (subseq query-string idx (1+ idx))))
+ (cond ((string= "(" current-char)
+ (setf open-brackets (1+ open-brackets))
+ (if (progn-p query-string idx)
+ (push-string "(progn " filter-string)
+ (push-string current-char filter-string)))
+ ((string= ")" current-char)
+ (setf open-brackets (1- open-brackets))
+ (when (< open-brackets 0)
+ (make-sparql-parser-condition
+ (subseq query-string idx)
+ (original-query construct)
+ "an opening bracket \"(\" is missing for the current closing one"))
+ (push-string current-char filter-string))
+ ((or (string= "'" current-char)
+ (string= "\"" current-char))
+ (let ((result (get-literal (subseq query-string idx))))
+ (unless result
+ (make-sparql-parser-condition
+ (subseq query-string idx)
+ (original-query construct)
+ "a closing character for the given literal"))
+ (setf idx (- (1- (length query-string))
+ (length (getf result :next-query))))
+ (push-string (getf result :literal) filter-string)))
+ ((string= "#" current-char)
+ (let ((comment-string
+ (string-until (subseq query-string idx)
+ (string #\newline))))
+ (setf idx (+ idx (length comment-string)))))
+ ((and (string= current-char (string #\newline))
+ (= 0 open-brackets))
+ (setf result
+ (list :next-query (subseq query-string idx)
+ :filter-string filter-string))
+ (setf idx (1- (length query-string))))
+ ((string= current-char "}")
+ (when (/= open-brackets 0)
+ (make-sparql-parser-condition
+ (subseq query-string idx)
+ (original-query construct)
+ "a valid filter, but the filter is not complete"))
+ (setf result
+ (list :next-query (subseq query-string idx)
+ :filter-string filter-string)))
+ (t
+ (push-string current-char filter-string)))))
+ result)))
+
+
+(defun progn-p(query-string idx)
+ "Returns t if the ( at position idx in the filter string
+ represents a (progn) block."
+ (declare (String query-string)
+ (Integer idx))
+ (let* ((delimiters (append (list " " (string #\Space) (string #\Tab)
+ (string #\Newline) (string #\cr) "(" ")")
+ *supported-operators*))
+ (string-before (trim-whitespace-right (subseq query-string 0 idx)))
+ (fragment-before-idx
+ (search-first delimiters string-before :from-end t))
+ (fragment-before
+ (if (and (not fragment-before-idx)
+ (and (> (length string-before) 0)
+ (not (find string-before *supported-functions*
+ :test #'string=))))
+ (error (make-condition
+ 'SPARQL-PARSER-ERROR
+ :message (format nil "Invalid filter: \"~a\"~%"
+ query-string)))
+ (when fragment-before-idx
+ (let ((inner-value
+ (subseq string-before fragment-before-idx)))
+ (if (and (> (length inner-value) 1)
+ (string-starts-with inner-value "("))
+ (subseq inner-value 1)
+ inner-value))))))
+ (if fragment-before
+ (progn
+ (when (or (string-starts-with fragment-before "?")
+ (string-starts-with fragment-before "$"))
+ (error
+ (make-condition
+ 'SPARQL-PARSER-ERROR
+ :message (format nil "Invalid filter: found \"~a\" but expected ~a"
+ fragment-before *supported-functions*))))
+ (when (not (find fragment-before (append *supported-functions*
+ delimiters)
+ :test #'string=))
+ (error
+ (make-condition
+ 'SPARQL-PARSER-ERROR
+ :message
+ (format nil "Invalid character: ~a, expected characters: ~a"
+ fragment-before (append *supported-functions* delimiters)))))
+ (if (find fragment-before *supported-functions* :test #'string=)
+ nil
+ t))
+ (if (find string-before *supported-functions* :test #'string=)
+ nil
+ t))))
+
+
+(defun get-literal (query-string)
+ "Returns a list of the form (:next-query <string> :literal <string>
+ where next-query is the query after the found literal and literal
+ is the literal string."
+ (declare (String query-string))
+ (cond ((or (string-starts-with query-string "\"\"\"")
+ (string-starts-with query-string "'''"))
+ (let ((literal-end
+ (find-literal-end (subseq query-string 3) (subseq query-string 0 3))))
+ (when literal-end
+ (list :next-query (subseq query-string (+ 3 literal-end))
+ :literal (concatenate 'string "'''"
+ (subseq query-string 3 literal-end)
+ "'''")))))
+ ((or (string-starts-with query-string "\"")
+ (string-starts-with query-string "'"))
+ (let ((literal-end
+ (find-literal-end (subseq query-string 1)(subseq query-string 0 1))))
+ (when literal-end
+ (list :next-query (subseq query-string (+ 1 literal-end))
+ :literal (concatenate 'string "'''"
+ (subseq query-string 1 literal-end)
+ "'''")))))))
+
(defun find-literal-end (query-string delimiter &optional (overall-pos 0))
"Returns the end of the literal corresponding to the passed delimiter
Modified: trunk/src/TM-SPARQL/sparql_parser.lisp
==============================================================================
--- trunk/src/TM-SPARQL/sparql_parser.lisp (original)
+++ trunk/src/TM-SPARQL/sparql_parser.lisp Tue Dec 14 16:07:50 2010
@@ -9,19 +9,6 @@
(in-package :TM-SPARQL)
-(defun make-sparql-parser-condition(rest-of-query entire-query expected)
- "Creates a spqrql-parser-error object."
- (declare (String rest-of-query entire-query expected))
- (let ((message
- (format nil "The query:~%\"~a\"~%~%has a bad token at position ~a => ~a.~%Expected: ~a"
- entire-query (- (length entire-query)
- (length rest-of-query))
- (subseq entire-query (- (length entire-query)
- (length rest-of-query)))
- expected)))
- (make-condition 'sparql-parser-error :message message)))
-
-
(defun parse-closed-value(query-string query-object &key (open "<") (close ">"))
"A helper function that checks the value of a statement within
two brackets, i.e. <prefix-value>. A list of the
Modified: trunk/src/base-tools/base-tools.lisp
==============================================================================
--- trunk/src/base-tools/base-tools.lisp (original)
+++ trunk/src/base-tools/base-tools.lisp Tue Dec 14 16:07:50 2010
@@ -150,16 +150,20 @@
nil)))
-(defun search-first (search-strings main-string)
+(defun search-first (search-strings main-string &key from-end)
"Returns the position of one of the search-strings. The returned position
is the one closest to 0. If no search-string is found, nil is returned."
(declare (String main-string)
- (List search-strings))
+ (List search-strings)
+ (Boolean from-end))
(let ((positions
- (remove-null (map 'list #'(lambda(search-str)
- (search search-str main-string))
- search-strings))))
- (let ((sorted-positions (sort positions #'<)))
+ (remove-null
+ (map 'list #'(lambda(search-str)
+ (search search-str main-string :from-end from-end))
+ search-strings))))
+ (let ((sorted-positions (if from-end
+ (sort positions #'>)
+ (sort positions #'<))))
(when sorted-positions
(first sorted-positions)))))
Modified: trunk/src/unit_tests/sparql_test.lisp
==============================================================================
--- trunk/src/unit_tests/sparql_test.lisp (original)
+++ trunk/src/unit_tests/sparql_test.lisp Tue Dec 14 16:07:50 2010
@@ -29,7 +29,8 @@
:test-set-result-3
:test-set-result-4
:test-set-result-5
- :test-result))
+ :test-result
+ :test-set-boundings))
(in-package :sparql-test)
@@ -1038,6 +1039,29 @@
(getf (first (result q-obj-2)) :result) :test #'string=)))))))))
+(test test-set-boundings
+ "Tests various cases of the function set-boundings"
+ (let* ((dummy-object (make-instance 'TM-SPARQL::SPARQL-Query :query " "))
+ (str-1 "BOUND(( (?var) )) || (isLITERAL($var) && ?var = 'abc')}")
+ (result-1 (tm-sparql::set-boundings dummy-object str-1))
+ (str-2
+ "(REGEX(?var1, '''''', ?var3) || (?var1 > ?var3 && (STR( ?var) = \"abc\")))}")
+ (result-2 (tm-sparql::set-boundings dummy-object str-2))
+ (str-3
+ "DATATYPE(?var3) || +?var1 = -?var2
+ ?var1 ?var2 ?var3}")
+ (result-3 (tm-sparql::set-boundings dummy-object str-3)))
+ (is-true result-1)
+ (is-true result-2)
+ (is (string= (getf result-1 :filter-string)
+ "BOUND((progn (progn ?var) )) || (progn isLITERAL($var) && ?var = '''abc''')"))
+ (is (string= (getf result-1 :next-query) "}"))
+ (is (string= (getf result-2 :filter-string)
+ "(progn REGEX(?var1, '''''', ?var3) || (progn ?var1 > ?var3 && (progn STR( ?var) = '''abc''')))"))
+ (is (string= (getf result-2 :next-query) "}"))
+ (is (string= (getf result-3 :filter-string)
+ "DATATYPE(?var3) || +?var1 = -?var2"))
+ (is (string= (getf result-3 :next-query) (subseq str-3 34)))))
(defun run-sparql-tests ()
