Librem-14 crash/freeze/hang

My new Librem-14 laptop LCD and HDMI screens froze.
Had to hold long-⏻ power key for hard shutdown.
Diagnostic (and other) advice appreciated!

Peace
	--Devon

P.S. MacOS saves crash data and presents it after reboot, including stack traces, active processes, etc. - does PureOS do this, and if so how can I see the latest crash data?

P.P.S. Hang coincided with Web/epiphany new-tab [+] click. Forgot to check CapsLock LED response, oops! Want the EC to watchdog the OS and blink Morse on the alarm LED when wedged.

P.P.P.S. Way too much detail follows:
# uname -a
Linux ******** 5.10.0-23-amd64 #1 SMP Debian 5.10.179-1 (2023-05-12) x86_64 GNU/Linux
# gnome-shell --version
GNOME Shell 3.38.6
# cat /etc/debian_version
buster/sid
# ~devon/bin/purism_ectool info
board: purism/librem_14
version: 1.13-devon_2023-03-22
# diff -Bbdur ~devon/{dist,hack}/Purism
Only in /home/devon/hack/Purism/librem-ec: build
Binary files /home/devon/dist/Purism/librem-ec/.git/index and /home/devon/hack/Purism/librem-ec/.git/index differ
diff -Bbdur /home/devon/dist/Purism/librem-ec/Makefile /home/devon/hack/Purism/librem-ec/Makefile
— /home/devon/dist/Purism/librem-ec/Makefile 2023-05-25 10:03:14.443239499 -0400
+++ /home/devon/hack/Purism/librem-ec/Makefile 2023-06-24 12:35:40.100333874 -0400
@@ -32,6 +32,9 @@
CFLAGS=-I$(COMMON_DIR)/include -D__FIRMWARE_VERSION__=$(VERSION)
include $(COMMON_DIR)/common.mk

+# Just a minute here - let the builder speak!
+CFLAGS+=$(EXTRA_CFLAGS)
+
 # Include the board's source
 BOARD_DIR=src/board/$(BOARD)
 SRC+=$(wildcard $(BOARD_DIR)/*.c)
diff -Bbdur /home/devon/dist/Purism/librem-ec/src/board/purism/librem_14/include/board/keymap.h /home/devon/hack/Purism/librem-ec/src/board/purism/librem_14/include/board/keymap.h
--- /home/devon/dist/Purism/librem-ec/src/board/purism/librem_14/include/board/keymap.h	2023-05-25 10:03:14.451239475 -0400
+++ /home/devon/hack/Purism/librem-ec/src/board/purism/librem_14/include/board/keymap.h	2023-06-24 11:38:15.884116195 -0400
@@ -53,7 +53,12 @@
 #define MATRIX_ESC_OUTPUT   10
 
 // Position of physical Fn key in the matrix
+#ifdef DEVON
+#define MATRIX_FN_INPUT     1
+#define MATRIX_FN_OUTPUT    3
+#else // DEVON
 #define MATRIX_FN_INPUT     2
 #define MATRIX_FN_OUTPUT    13
+#endif // DEVON
 
 #endif // _BOARD_KEYMAP_H
Only in /home/devon/hack/Purism/librem-ec/src/board/purism/librem_14/keymap: devon.c
# diff -Bbdur ~devon/hack/Purism/librem-ec/src/board/purism/librem_14/keymap/{default,devon}.c
--- /home/devon/hack/Purism/librem-ec/src/board/purism/librem_14/keymap/default.c	2023-05-25 10:03:14.000000000 -0400
+++ /home/devon/hack/Purism/librem-ec/src/board/purism/librem_14/keymap/devon.c	2023-06-24 19:06:38.915829593 -0400
@@ -1,26 +1,52 @@
 // SPDX-License-Identifier: GPL-3.0-only
+// librem-ec/src/board/purism/librem_14/keymap/devon.c
 
-// Default layout
+// Devonian layout
+// Swap CapsLock & BackSpace here, not in kmonad --
+//	kmonad-mapped CapsLock autorepeat fails in Mozilla.
+// Kmonad best customizes the key left of (tiny) RightShift --
+//	with tap/hold dual functions on the same key.
+// Swap LeftControl & Fn -- like on ThinkPads and MacBooks
+//	where the Fn key is in the lower left corner.
+// TO DO
+// Dynamic keymapping should support all such customizations --
+//	without breaking the Fn+Esc reset function.
+// Fn+F1 & Fn+F2 should do something useful.
+// Should Fn+Space duplicate Fn+F4 cycle keyboard backlight glow?
+// Solve FireFox & Thunderbird physical CapsLock autorepeat failure.
 
 #include <board/keymap.h>
+#ifdef DEVON
+  #define BKSP_CAPS K_CAPS
+  #define CAPS_BKSP K_BKSP
+  #define LCTRL_FN KT_FN
+  #define FN_LCTRL K_LEFT_CTRL
+#else // DEVON
+  #define BKSP_CAPS K_BKSP
+  #define CAPS_BKSP K_CAPS
+  #define LCTRL_FN K_LEFT_CTRL
+  #define FN_LCTRL KT_FN
+#endif // DEVON
 
 uint16_t __code KEYMAP[KM_LAY][KM_OUT][KM_IN] = {
 // regular key layout
 LAYOUT(
     K_ESC, K_F1, K_F2, K_F3, K_F4, K_F5, K_F6, K_F7, K_F8, K_F9, K_F10, K_F11, K_F12, K_PRINT_SCREEN, K_DEL,
-    K_TICK, K_1, K_2, K_3, K_4, K_5, K_6, K_7, K_8, K_9, K_0, K_MINUS, K_EQUALS, K_BKSP,
+    K_TICK, K_1, K_2, K_3, K_4, K_5, K_6, K_7, K_8, K_9, K_0, K_MINUS, K_EQUALS, BKSP_CAPS,
     K_TAB, K_Q, K_W, K_E, K_R, K_T, K_Y, K_U, K_I, K_O, K_P, K_BRACE_OPEN, K_BRACE_CLOSE, K_BACKSLASH,
-    K_CAPS, K_A, K_S, K_D, K_F, K_G, K_H, K_J, K_K, K_L, K_SEMICOLON, K_QUOTE, K_ENTER,
+    CAPS_BKSP, K_A, K_S, K_D, K_F, K_G, K_H, K_J, K_K, K_L, K_SEMICOLON, K_QUOTE, K_ENTER,
     K_LEFT_SHIFT, K_Z, K_X, K_C, K_V, K_B, K_N, K_M, K_COMMA, K_PERIOD, K_SLASH, K_UP, K_RIGHT_SHIFT,
-    K_LEFT_CTRL, KT_FN, K_LEFT_SUPER, K_LEFT_ALT, K_SPACE, K_RIGHT_ALT, K_RIGHT_CTRL, K_LEFT, K_DOWN, K_RIGHT
+    LCTRL_FN, FN_LCTRL, K_LEFT_SUPER, K_LEFT_ALT, K_SPACE, K_RIGHT_ALT, K_RIGHT_CTRL, K_LEFT, K_DOWN, K_RIGHT
 ),
 // FN+key layout
 LAYOUT(
     K_SUSPEND, K_F1, K_F2, K_DISPLAY_MODE, K_KBD_BKL, K_BRIGHTNESS_DOWN, K_BRIGHTNESS_UP, K_MEDIA_PREV, K_PLAY_PAUSE, K_MEDIA_NEXT, K_MUTE, K_VOLUME_DOWN, K_VOLUME_UP, K_PRINT_SCREEN, K_INSERT,
-    K_TICK, K_1, K_2, K_3, K_4, K_5, K_6, K_7, K_8, K_9, K_0, K_MINUS, K_EQUALS, K_BKSP,
+    K_TICK, K_1, K_2, K_3, K_4, K_5, K_6, K_7, K_8, K_9, K_0, K_MINUS, K_EQUALS, BKSP_CAPS,
     K_TAB, K_Q, K_W, K_E, K_R, K_T, K_Y, K_U, K_I, K_O, K_P, K_BRACE_OPEN, K_BRACE_CLOSE, K_BACKSLASH,
-    K_CAPS, K_A, K_S, K_D, K_F, K_G, K_H, K_J, K_K, K_L, K_SEMICOLON, K_QUOTE, K_ENTER,
+    CAPS_BKSP, K_A, K_S, K_D, K_F, K_G, K_H, K_J, K_K, K_L, K_SEMICOLON, K_QUOTE, K_ENTER,
     K_LEFT_SHIFT, K_Z, K_X, K_C, K_V, K_B, K_N, K_M, K_COMMA, K_PERIOD, K_SLASH, K_PGUP, K_RIGHT_SHIFT,
-    K_LEFT_CTRL, KT_FN, K_LEFT_SUPER, K_LEFT_ALT, K_KBD_BKL, K_RIGHT_ALT, K_RIGHT_CTRL, K_HOME, K_PGDN, K_END
+    LCTRL_FN, FN_LCTRL, K_LEFT_SUPER, K_LEFT_ALT, K_KBD_BKL, K_RIGHT_ALT, K_RIGHT_CTRL, K_HOME, K_PGDN, K_END
 )
 };
+
+// end librem-ec/src/board/purism/librem_14/keymap/devon.c
\ No newline at end of file

I would say: new device => contact Purism Support. support@puri.sm

1 Like

I contacted support - and also hope to learn from our collective wisdom.

2 Likes

Froze again but this time I had a process blinking the RGB LED, which kept blinking so I suspect only the GPU was wedged. I let it sit for an hour or so, found it snoring and woke it ok, suggesting Fn-Esc as a possible (but obnoxious) workaround. I now have enabled SSH and run two processes writing to the RGB LED, one cycles blue/green at 0.2 Hz, the other flashes red when I whip the mouse.

POSSIBLE WORKAROUND

Symptom - Librem-14 screens, mouse cursor, etc., crash/freeze/hang.

Tap Fn-Esc - should instantly sleep but nothing happens ## showing the system is unresponsive.

Tap Fn-F4 six times - keyboard glow backlight intensity should change ## showing the Embedded Controller (EC) responds.

Wait - long enough to launch an ssh session into the Librem-14 and type $ top -n1 ## showing Xwayland burning 100% CPU.

The fans roar. If you’re lucky, after a while the screens go black, ssh stops responding and the battery LED starts snoring ## showing that Fn-Esc eventually slept the system after all.

Tap Power to wake the sleeping system - everything seems normal until the next crash/freeze/hang which could be minutes away.

	Peace
		--Devon

P.S. Reporting these frequent crashes of a brand-new Librem-14 to Purism got advice to wait a few months for the next PureOS release. Looks like an Xwayland bug which was fixed last winter…

# apt source Xwayland
Reading package lists... Done
E: Unable to find a source package for Xwayland
# Xwayland --version
Unrecognized option: --version
# strings $(type -p Xwayland) | grep -i '\<version\>'
glamor%d: Failed to get GLSL version
glamor%d: Failed to parse GLSL version string %s
Require OpenGL version 2.1 or later.
#version 130
#version %d
Protocol version mismatch
.gnu.version

…stay tuned.

How unusual. What happens if you boot a PureOS live image on a USB drive instead? Are Coreboot and the EC firmware updated to the latest version?

1 Like

Unclear whether this workaround requires Fn-Esc, lid closure, or both.
I now question closing the lid - stopping the fans risks heat damage,
a possible EC bug - better spin the fans regardless of lid closure.
On the other hand, just powering down the CPU could be safer.

Every prior hang has been preceded by a “Firefox is not responding” popup
but today’s hang also had a “Gnu Emacs Client is not responding” popup.
As usual, Fn-F4 changes keyboard glow but Fn-Esc is unresponsive.

2023v30w1119 *	lid close, wait, open, snoring, login normally but fans screaming
2023v22t1450	devon    tty2         tty2             Tue Aug 22 14:50 - crash (1+23:55)
2023v15t1947 *	lid close, wait, open, snoring, login normally
2023v07m1150 *	dies again two hours later
2023v07m0940 *	lid close, wait, open, snoring, login normally
2023v07m0928 *	lid close, wait, open, snoring, login normally
2023f25n1136	devon    tty2         tty2             Sun Jun 25 11:36 - crash (13+04:58)
2023f25n1133 *	Fn-Esc unresponsive, hold power button to force reboot
2023f24s1335	devon    tty2         tty2             Sat Jun 24 13:35 - crash  (21:59)
2023f22r2126	devon    tty2         tty2             Thu Jun 22 21:26 - crash (1+16:09)
2023e31w2215	devon    tty2         tty2             Wed May 31 22:15 - crash (6+14:16)
2023e29m2113	devon    tty2         tty2             Mon May 29 21:13 - crash (1+21:54)

* indicates manually logged system hangs, other lines from $ last | grep 'crash'

You should not be experiencing anything like this with a new Librem 14 device. It will be a while before the next PureOS version, Crimson, will be released.

Use these links as a guide to reflash the Coreboot firmware. Create a PureOS live image on a USB drive as part of the flashing process, and use it for any further debugging needs after you are done.

https://downloads.puri.sm/10/gnome/2023-06-14/

Hung again just now at 13:17:50 and Fn-Esc followed by a ten minute wait did nothing, closing the lid for five minutes got it snoring and revived. Awfully unlikely that anything in the EC or boot process would affect this well-documented, already-fixed Xwayland bug. Nothing from # apt search xwayland-dev and I have no idea how to find out what build options work for PureOS.

	Peace
		--Devon

P.S. Will try more after backup. Too bad I’m not allowed to search https://source.puri.sm.
I had to update the EC to make the 3.5 mm audio jack work. I also swapped Ctrl↔Fn and CapsLock↔Backspace, I thought I posted my keymap patch but I don’t see it.
# ~devon/bin/purism_ectool info
board: purism/librem_14
version: 1.13-devon_2023-03-22
# dmidecode --string bios-version
4.19-Purism-2

My suggestion is to flash the PureOS live image onto a USB drive. You are free to verify it with the link I provided above. Otherwise consider experimenting with different operating systems instead to ensure that your issue can be resolved elsewhere.