NVMe: shutdown all controllers and fix IRQ handler

drivers/nvme/macros.inc

@@ -1,30 +0,0 @@
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;                                                                 ;;
-;; Copyright (C) KolibriOS team 2004-2024. All rights reserved.    ;;
-;; Distributed under terms of the GNU General Public License       ;;
-;;                                                                 ;;
-;;          GNU GENERAL PUBLIC LICENSE                             ;;
-;;             Version 2, June 1991                                ;;
-;;                                                                 ;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-macro PDEBUGF _level*, _fmt*, _bus*, _devfn*, [_args] {
-	common
-	if __DEBUG__
-		sub 	esp, 12
-		push 	ebx
-		movzx 	ebx, _bus
-		mov     dword [esp + 4], ebx
-		movzx   ebx, _devfn
-		shr     ebx, 3 ; get rid of 3 lowest bits (function code), the rest bits is device code
-		mov 	dword [esp + 8], ebx
-		movzx   ebx, _devfn
-		and     ebx, 00000111b ; get only 3 lowest bits (function code)
-		mov 	dword [esp + 12], ebx
-		pop 	ebx
-		DEBUGF  _level, _fmt, [esp], [esp + 4], [esp + 8], _args
-		add 	esp, 12
-	end if
-}
-
-; vim: syntax=fasm

drivers/nvme/nvme.asm

@@ -29,7 +29,6 @@ include "../fdo.inc"
 include "../pci.inc"
 include "../peimport.inc"
 include "nvme.inc"
-include "macros.inc"
 include "lib.inc"
 include "command.inc"
 
@@ -77,7 +76,7 @@ local   AnythingLoadedSuccessfully db 0
 	test 	eax, eax
 	setne	[AnythingLoadedSuccessfully]
 	inc 	ebx
-	cmp 	ebx, dword [pcidevs_len]
+	cmp 	ebx, dword [num_pcidevs]
 	jne 	.loop
 	cmp 	[AnythingLoadedSuccessfully], 0
 	jz 	.err
@@ -537,7 +536,7 @@ proc nvme_readwrite stdcall, ns:dword, buf:dword, start_sector:qword, numsectors
 endp
 
 ; Detects NVMe devices on the PCI bus and stores them into 
-; [p_nvme_devices] and sets [pcidevs_len] to the appropriate
+; [p_nvme_devices] and sets [num_pcidevs] to the appropriate
 ; size based off how many NVMe devices there are.
 proc detect_nvme
 
@@ -572,14 +571,14 @@ proc detect_nvme
 	jnz 	.err
 
 @@:
-	PDEBUGF DBG_INFO, "PCI(%u.%u.%u): Detected NVMe device...\n", [esi + PCIDEV.bus], [esi + PCIDEV.devfn]
+	cmp 	dword [num_pcidevs], TOTAL_PCIDEVS
-	cmp 	dword [pcidevs_len], TOTAL_PCIDEVS
 	jne 	@f
 	DEBUGF  DBG_INFO, "Can't add any more NVMe devices...\n"
 	jmp 	.exit_success
 
 @@:
-	inc 	dword [pcidevs_len]
+	inc 	dword [num_pcidevs]
+	add 	dword [num_pcidevs_sz], sizeof.pcidev
 	cmp     dword [p_nvme_devices], 0
 	jnz	@f ; was the pointer already allocated?
 	invoke  KernelAlloc, sizeof.pcidev * TOTAL_PCIDEVS
@@ -590,7 +589,7 @@ proc detect_nvme
 	DEBUGF  DBG_INFO, "nvme: Allocated memory for PCI devices at: 0x%x\n", eax
 
 @@:
-	mov 	ecx, dword [pcidevs_len]
+	mov 	ecx, dword [num_pcidevs]
 	dec 	ecx
 	mov 	edi, dword [p_nvme_devices]
 	mov 	edx, ecx
@@ -644,7 +643,7 @@ proc device_is_compat stdcall, pci:dword
 	ret
 
 .failure:
-	PDEBUGF  DBG_INFO, "PCI(%u.%u.%u): something went wrong checking NVMe device compatibility\n", byte [esi + pcidev.bus], byte [esi + pcidev.devfn]
+	DEBUGF  DBG_INFO, "nvme%u: something went wrong checking NVMe device compatibility\n", [esi + pcidev.num]
 	pop 	 ecx edx esi
 	xor      eax, eax
 	ret
@@ -1138,6 +1137,12 @@ proc cqyhdbl_write stdcall, pci:dword, y:dword, cqh:dword
 	mov 	word [esi + edx], ax ; Write to CQyHDBL
 	mov 	word [edi + NVM_QUEUE_ENTRY.head], ax
 
+	; NOTE: Currently commented out since we're just using
+	; plain spinlocks for notifying when a command has been
+	; completed, but this will be uncommented later and use
+	; semaphores instead of mutexes once the polling code
+	; has been replaced with the asynchronous API.
+
 	; Unlock the mutex now that the command is complete
 	;mov 	edi, dword [edi + NVM_QUEUE_ENTRY.cmd_ptr]
 	;mov 	ecx, [cqh]
@@ -1277,19 +1282,24 @@ endp
 proc irq_handler
 
 	push 	ebx esi edi
-	mov 	edi, dword [p_nvme_devices]
+	mov 	esi, dword [p_nvme_devices]
-	mov 	esi, edi
+	mov 	ebx, dword [num_pcidevs_sz]
-	sub 	esi, sizeof.pcidev
+	add 	ebx, esi
-	mov 	ebx, dword [pcidevs_len]
-	xor 	ecx, ecx
 
 .check_who_raised_irq:
+	stdcall device_generated_interrupt, esi
+	test 	eax, eax
+	jnz 	@f
 	add 	esi, sizeof.pcidev
-	inc 	ecx
+	cmp 	esi, ebx
-	cmp 	ecx, ebx
+	jbe 	.check_who_raised_irq
-	; TODO: Apply solution given by @punk_joker of checking which device 
+
-	; generated an interrupt.
+	; Interrupt not handled by driver, return 0
-	ja 	.not_our_irq
+	pop 	edi esi ebx
+	xor 	eax, eax
+	ret
+
+@@:
 	mov 	edi, dword [esi + pcidev.io_addr]
 	mov 	dword [edi + NVME_MMIO.INTMS], 0x3
 	stdcall consume_cq_entries, esi, ADMIN_QUEUE
@@ -1303,12 +1313,28 @@ proc irq_handler
 	mov 	eax, 1
 	ret
 
-.not_our_irq:
+endp
-	; Interrupt not handled by driver, return 0
+
-	pop 	edi esi ebx
+proc device_generated_interrupt stdcall, pci:dword
+
+	mov 	edx, [pci]
+	mov 	edx, dword [edx + pcidev.queue_entries]
+	xor 	ecx, ecx
+
+@@:
+	mov 	ax, word [edx + ecx + NVM_QUEUE_ENTRY.head]
+	cmp 	ax, word [edx + ecx + NVM_QUEUE_ENTRY.tail]
+	jne 	@f
+	add 	ecx, sizeof.NVM_QUEUE_ENTRY
+	cmp 	ecx, LAST_QUEUE_ID * sizeof.NVM_QUEUE_ENTRY
+	jbe 	@b
 	xor 	eax, eax
 	ret
 
+@@:
+	mov 	eax, 1
+	ret
+
 endp
 
 ; Deletes the allocated I/O queues for all of the NVMe devices,
@@ -1358,13 +1384,7 @@ proc nvme_cleanup
 	cmp 	ebx, LAST_QUEUE_ID
 	jbe 	.get_queue
 	pop 	ebx
-	inc 	ebx
-	cmp 	ebx, dword [pcidevs_len]
-	jne 	.get_pcidev
 
-	; NOTE: This code has a bug! It only shuts down the last
-	; controller, not all of them. Move this inside the loop
-	; and check if the device is actually valid.
 	; Shutdown the controller
 	mov 	edi, dword [esi + pcidev.io_addr]
 	mov 	eax, dword [edi + NVME_MMIO.CC]
@@ -1378,6 +1398,10 @@ proc nvme_cleanup
 	test 	byte [edi + NVME_MMIO.CSTS], CSTS_SHST_SHUTDOWN_COMPLETE
 	jnz 	@b
 
+	inc 	ebx
+	cmp 	ebx, dword [num_pcidevs]
+	jne 	.get_pcidev
+
 .ret:
 	pop 	edi esi ebx
 	ret
@@ -1386,8 +1410,9 @@ endp
 
 ;all initialized data place here
 align 4
-	p_nvme_devices dd 0
+	p_nvme_devices dd 0 ; Pointer to array of NVMe devices
-	pcidevs_len  dd 0
+	num_pcidevs    dd 0 ; Number of NVMe devices
+	num_pcidevs_sz dd 0 ; Size in bytes
 	my_service   db "nvme",0  ;max 16 chars include zero
 	disk_functions:
 		dd 	 disk_functions.end - disk_functions