GitLab

at 2023-01-13T23:33:47+00:00

Address #139: Set filename encoding to :utf-8

Merge branch 'issue-139-set-filename-encoding-to-utf8' into 'master'

Address #139: Set filename encoding to :utf-8

See merge request cmucl/cmucl!109
Fix #162:  Change *filename-encoding* to use :no-encoding

Instead of using `NIL` to indicate that `*filename-encoding*` is not
to be done, use `:no-encoding` to indicate that.  This makes it a bit
clearer what `*filename-encoding*` means.

Use :null instead of :no-encoding for no filename encoding

The advantage of using `:null` is that it's a recognized external
format (that aliases to `:void`).  So if we inadvertently use `:null`
as a filename encoding somewhere unexpected, it will cause an
error (because the `:void` encoding does).

Fix stupid typos

Really stupid typos/thinkos:
* Forgot to change initial value if `*filename-encoding*` from
  `:no-encoding` to `:null`.  (Stupid!)
* Fix typo: `:nul` instead of `:null` in `%name->file`.

Update cmucl-unix.pot too for the change in the docstring for
`*filename-encoding*`.

Disable issue.41.1 when running CI

This test was previously disabled only for Linux when running the CI.
However, it's now also failing when running the CI for Darwin.  Thus
disable it whenever we're running the CI.

I just manually tested this on my Linux and Mac boxes.  This test
passes without any problem.  Not sure what's going on.

Merge branch 'issue-162-filename-encoding-no-encoding' into 'master'

Fix #162:  Change *filename-encoding* to use :null

Closes #162

See merge request cmucl/cmucl!111
Merge branch 'master' into issue-158-darwin-pathnames

 		    #() '())))))

 (defun load-external-format-aliases ()

+  ;; Set filename encoding to NIL to bypass any encoding; it's not

+  ;; needed to open the aliases file.  NIL means the pathname string is passed as is where only the low 8 bits of the 

   (let ((*package* (find-package "KEYWORD"))

-	(unix::*filename-encoding* :iso8859-1))

+	(unix::*filename-encoding* :null))

     (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil

 			 :external-format :iso8859-1)

       (when stm

       (and (consp name) (find-external-format name))

       (and (with-standard-io-syntax

 	     ;; Use standard IO syntax so that changes by the user

-	     ;; don't mess up compiling the external format.

-	     (let ((*package* (find-package "STREAM"))

-		   (lisp::*enable-package-locked-errors* nil)

-		   (s (open (format nil "ext-formats:~(~A~).lisp" name)

-			    :if-does-not-exist nil :external-format :iso8859-1)))

+	     ;; don't mess up compiling the external format, but we

+	     ;; don't need to print readably.  Also, set filename

+	     ;; encoding to NIL because we don't need any special

+	     ;; encoding to open the format files.

+	     (let* ((*print-readably* nil)

+		    (unix::*filename-encoding* :null)

+		    (*package* (find-package "STREAM"))

+		    (lisp::*enable-package-locked-errors* nil)

+		    (s (open (format nil "ext-formats:~(~A~).lisp" name)

+			     :if-does-not-exist nil :external-format :iso8859-1)))

 	       (when s

 		 (null (nth-value 1 (ext:compile-from-stream s))))))

            (gethash name *external-formats*))))

     (unless (find-external-format filenames)

       (error (intl:gettext "Can't find external-format ~S.") filenames))

     (setq filenames (ef-name (find-external-format filenames)))

-    (when (and unix::*filename-encoding*

+    (when (and (not (eq unix::*filename-encoding* :null))

 	       (not (eq unix::*filename-encoding* filenames)))

       (cerror (intl:gettext "Change it anyway.")

 	      (intl:gettext "The external-format for encoding filenames is already set.")))

   #-gengc (setf unix::*interrupt-pending* nil)

   (setf *type-system-initialized* nil)

   (setf *break-on-signals* nil)

-  (setf unix::*filename-encoding* nil)

+  (setf unix::*filename-encoding* :null)

   (setf *enable-darwin-path-normalization* nil)

   #+gengc (setf conditions::*handler-clusters* nil)

   (setq intl::*default-domain* "cmucl")

 		 *default-external-format*))))

   (values))

- 

+(defun decode-runtime-strings (locale file-locale)

+  ;; The C runtime can initialize the following strings from the

+  ;; command line or the environment.  We need to decode these into

+  ;; the utf-16 strings that Lisp uses.

+  (setf lisp-command-line-list

+	(mapcar #'(lambda (s)

+		    (stream:string-decode s locale))

+		lisp-command-line-list))

+  (setf lisp-environment-list

+	(mapcar #'(lambda (s)

+		    (stream:string-decode s locale))

+		lisp-environment-list))

+  ;; This needs more work..  *cmucl-lib* could be set from the the envvar

+  ;; "CMUCLLIB" or from the "-lib" command-line option, and thus

+  ;; should use the LOCALE to decode the string.

+  (when *cmucl-lib*

+    (setf *cmucl-lib*

+	  (stream:string-decode *cmucl-lib* file-locale)))

+  ;; This also needs more work since the core path could come from the

+  ;; "-core" command-line option and should thus use LOCALE to decode

+  ;; the string.  It could also come from the "CMUCLCORE" envvar.

+  (setf *cmucl-core-path*

+	(stream:string-decode *cmucl-core-path* file-locale))

+  ;; *unidata-path* defaults to a pathname object, but the user can

+  ;; specify a path, so we need to decode the string path if given.

+  (when (and *unidata-path* (stringp *unidata-path*))

+    (setf *unidata-path*

+	  (stream:string-decode *unidata-path* file-locale))))

+

 (defun save-lisp (core-file-name &key

 				 (purify t)

 				 (root-structures ())

 	     ;; Load external format aliases now so we can aliases to

 	     ;; specify the external format.

 	     (stream::load-external-format-aliases)

-	     ;; Set the locale for lisp

-	     (intl::setlocale)

 	     ;; Set up :locale format

 	     (set-up-locale-external-format)

+<<<<<<< HEAD

 	     ;; Set terminal encodings to :locale

 	     (set-system-external-format :locale)

 	     #+darwin

 	       (lisp::load-decomp)

 	       (lisp::load-combining)

 	       (setf *enable-darwin-path-normalization* t))

+=======

+	     ;; Set terminal encodings to :locale and filename encoding to :utf-8.

+	     ;; (This needs more work on Darwin.)

+	     (set-system-external-format :locale :utf-8)

+	     (decode-runtime-strings :locale :utf-8)

+	     ;; Need to reinitialize the environment again because

+	     ;; we've possibly changed the environment variables and

+	     ;; pathnames.

+	     (environment-init)

+	     ;; Set the locale for lisp

+	     (intl::setlocale)

+>>>>>>> master

 	     (ext::process-command-strings process-command-line)

 	     (setf *editor-lisp-p* nil)

 	     (macrolet ((find-switch (name)

 ;; it must be set to :iso8859-1 (or left as NIL), making files with

 ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible.

 ;; Must be set to NIL initially to enable building Lisp!

-(defvar *filename-encoding* nil)

+(defvar *filename-encoding* :null

+  "The encoding to use for converting a namestring to a string that can

+  be used by the operations system.  It must be a valid

+  external-format name or :NULL.  :NULL means the string

+  is passed as is to the operating system.  The operating system will

+  get the low 8 bits of each UTF-16 code unit of the string.")

 (eval-when (:compile-toplevel :load-toplevel :execute)

   (defmacro %name->file (string)

-    `(if *filename-encoding*

-	 (string-encode ,string *filename-encoding*)

-	 ,string))

+    `(if (eql *filename-encoding* :null)

+	 ,string

+	 (string-encode ,string *filename-encoding*)))

   (defmacro %file->name (string)

-    `(if *filename-encoding*

-	 (string-decode ,string *filename-encoding*)

-	 ,string)))

+    `(if (eql *filename-encoding* :null)

+	 ,string

+	 (string-decode ,string *filename-encoding*))))

 ;;;; Common machine independent structures.

 "Content-Type: text/plain; charset=UTF-8\n"

 "Content-Transfer-Encoding: 8bit\n"

+#: src/code/unix.lisp

+msgid ""

+"The encoding to use for converting a namestring to a string that can\n"

+"  be used by the operations system.  It must be a valid\n"

+"  external-format name or :NULL.  :NULL means the string\n"

+"  is passed as is to the operating system.  The operating system will\n"

+"  get the low 8 bits of each UTF-16 code unit of the string."

+msgstr ""

+

 #: src/code/unix.lisp

 msgid "Syscall ~A failed: ~A"

 msgstr ""

 	(assert-equal (map 'list #'char-code out-string)

 		      (map 'list #'char-code expected))))))

+(define-test issue.25c-setup

+    (:tag :issues)

+  ;; Get the external format before running the test issue.25c.  See

+  ;; issue #161

+  ;; (https://gitlab.common-lisp.net/cmucl/cmucl/-/issues/161).

+  (assert-true (stream::find-external-format :utf16-be)))

+

 (define-test issue.25c

     (:tag :issues)

   ;; Modified test to verify that each octet read from run-program is

 ;; running a pipeline with linux, but otherwise enable it.  The

 ;; pipeline defines the envvar GITLAB_CI so check for that.

 ;;

+;; This also fails on Darwin CI now.  Let's just disable the test if

+;; running on CI.

+;;

 ;; It would be better if lisp-unit had a way of marking tests as known

 ;; failures, but it doesn't.

-#+#.(cl:if (cl:and (ext:featurep :linux) (unix:unix-getenv "GITLAB_CI")) '(or) '(and))

+#+#.(cl:if (cl:and (unix:unix-getenv "GITLAB_CI")) '(or) '(and))

 (define-test issue.41.1

     (:tag :issues)

   (issue-41-tester unix:sigstop))

   ;; work and not return NIL.

   (assert-true (file-author "."))

   (assert-true (file-author "bin/build.sh"))

-  (let ((unix::*filename-encoding* :utf-8))

-    ;; Set filename encoding to utf-8 so that we can encode the

-    ;; filename properly.

-    (assert-true

+  (assert-true

    (file-author

     (merge-pathnames 

      (concatenate 'string

 		  '(#\Hangul_Syllable_An #\Hangul_Syllable_Nyeong #\Hangul_Syllable_Ha

 		    #\Hangul_Syllable_Sib #\Hangul_Syllable_Ni #\Hangul_Syllable_Gga)

 		  ".txt")

-     *test-path*)))))

+     *test-path*))))

 (define-test issue.139-default-external-format

     (:tag :issues)

Raymond Toy pushed to branch issue-158-darwin-pathnames at cmucl / cmucl

Commits:

6 changed files:

Changes: