mirror of
https://git.missingno.dev/kolibrios-nvme-driver/
synced 2026-03-09 12:53:24 +00:00
significant refactoring
* Removed useless `nvme_wait` call * Remove phase tag bit check in favor of spinlock checking * Added back in queue wrapping (though, it still needs work) * Fix in `is_queue_full` which didn't pop bx off the stack when returning in some circumstances
This commit is contained in:
@@ -192,6 +192,7 @@ endp
|
||||
; See pages 161-205 of the NVMe 1.4 specification for reference
|
||||
proc nvme_identify stdcall, pci:dword, nsid:dword, prp1:dword, cns:byte
|
||||
|
||||
LOCK_SPINLOCK
|
||||
sub esp, sizeof.SQ_ENTRY
|
||||
; It's important to check if CNS is a valid value here. In revision 1.0
|
||||
; CNS is a 1 bit field and a two bit field in revision 1.1, using invalid
|
||||
@@ -237,6 +238,7 @@ proc nvme_identify stdcall, pci:dword, nsid:dword, prp1:dword, cns:byte
|
||||
stdcall sqytdbl_write, [pci], ADMIN_QUEUE, esp
|
||||
|
||||
add esp, sizeof.SQ_ENTRY
|
||||
call nvme_poll
|
||||
ret
|
||||
|
||||
endp
|
||||
@@ -250,6 +252,8 @@ proc create_namespace stdcall, pci:dword, cid:word
|
||||
jz .fail
|
||||
invoke GetPhysAddr
|
||||
stdcall nvme_identify, [pci], 0xffffffff, eax, CNS_IDNS
|
||||
test eax, eax
|
||||
jz .fail
|
||||
|
||||
.fail:
|
||||
pop esi
|
||||
@@ -272,6 +276,8 @@ proc is_active_namespace stdcall, pci:dword, nsid:dword
|
||||
mov esi, eax
|
||||
invoke GetPhysAddr
|
||||
stdcall nvme_identify, [pci], [nsid], eax, CNS_IDNS
|
||||
test eax, eax
|
||||
jz .not_active_nsid
|
||||
xor ecx, ecx
|
||||
|
||||
@@:
|
||||
@@ -335,6 +341,7 @@ endp
|
||||
; See page 101 of the NVMe 1.4 specification for reference
|
||||
proc create_io_completion_queue stdcall, pci:dword, prp1:dword, qid:dword, ien:byte
|
||||
|
||||
LOCK_SPINLOCK
|
||||
sub esp, sizeof.SQ_ENTRY
|
||||
stdcall memsetdz, esp, sizeof.SQ_ENTRY / 4
|
||||
stdcall set_cdw0, [pci], ADMIN_QUEUE, ADM_CMD_CRE_IO_COMPLETION_QUEUE
|
||||
@@ -350,6 +357,7 @@ proc create_io_completion_queue stdcall, pci:dword, prp1:dword, qid:dword, ien:b
|
||||
mov dword [esp + SQ_ENTRY.cdw11], eax
|
||||
stdcall sqytdbl_write, [pci], ADMIN_QUEUE, esp
|
||||
add esp, sizeof.SQ_ENTRY
|
||||
call nvme_poll
|
||||
ret
|
||||
|
||||
endp
|
||||
@@ -357,6 +365,7 @@ endp
|
||||
; See page 103-104 of the NVMe 1.4 specification for reference
|
||||
proc create_io_submission_queue stdcall, pci:dword, prp1:dword, qid:dword, cqid:word
|
||||
|
||||
LOCK_SPINLOCK
|
||||
sub esp, sizeof.SQ_ENTRY
|
||||
stdcall memsetdz, esp, sizeof.SQ_ENTRY / 4
|
||||
stdcall set_cdw0, [pci], ADMIN_QUEUE, ADM_CMD_CRE_IO_SUBMISSION_QUEUE
|
||||
@@ -373,6 +382,7 @@ proc create_io_submission_queue stdcall, pci:dword, prp1:dword, qid:dword, cqid:
|
||||
mov dword [esp + SQ_ENTRY.cdw11], eax
|
||||
stdcall sqytdbl_write, [pci], ADMIN_QUEUE, esp
|
||||
add esp, sizeof.SQ_ENTRY
|
||||
call nvme_poll
|
||||
ret
|
||||
|
||||
endp
|
||||
@@ -670,7 +680,7 @@ proc nvme_readwrite stdcall, ns:dword, buf:dword, start_sector:qword, numsectors
|
||||
; to the allocated PRP list (after this call, only if it completes successfully)
|
||||
stdcall alloc_dptr, esi, ebx, eax, ecx, [buf]
|
||||
test eax, eax
|
||||
jz .dptr_fail
|
||||
jz .fail
|
||||
|
||||
DEBUGF DBG_INFO, "PRP1: %x, PRP2: %x\n", [ebx], [ebx + 4]
|
||||
mov eax, dword [start_sector]
|
||||
@@ -682,6 +692,8 @@ proc nvme_readwrite stdcall, ns:dword, buf:dword, start_sector:qword, numsectors
|
||||
mov ecx, dword [ebx + 12]
|
||||
dec ecx
|
||||
|
||||
; TODO: add non-blocking mechanisms later on
|
||||
LOCK_SPINLOCK
|
||||
stdcall nvme_io_rw, [esi + NSINFO.pci], \
|
||||
1, \
|
||||
[esi + NSINFO.nsid], \
|
||||
@@ -693,26 +705,25 @@ proc nvme_readwrite stdcall, ns:dword, buf:dword, start_sector:qword, numsectors
|
||||
dword [ebx + 8]
|
||||
|
||||
; free PRP list (if allocated)
|
||||
mov eax, dword [ebx + 16]
|
||||
test eax, eax
|
||||
mov edx, dword [ebx + 16]
|
||||
test edx, edx
|
||||
jz @f
|
||||
invoke KernelFree, eax
|
||||
invoke KernelFree, edx
|
||||
|
||||
@@:
|
||||
; assume command completes successfully for now
|
||||
call nvme_poll
|
||||
test eax, eax
|
||||
jz .fail
|
||||
xor eax, eax
|
||||
jmp .end
|
||||
|
||||
.dptr_fail:
|
||||
mov ebx, [numsectors_ptr]
|
||||
mov dword [ebx], 0
|
||||
or eax, -1 ; generic disk error
|
||||
|
||||
.end:
|
||||
add esp, 20
|
||||
pop edi esi ebx
|
||||
ret
|
||||
|
||||
.fail:
|
||||
mov ebx, [numsectors_ptr]
|
||||
mov dword [ebx], 0
|
||||
or eax, -1 ; generic disk error
|
||||
|
||||
endp
|
||||
|
||||
; See page 258-261 (read) and 269-271 (write) of the NVMe 1.4 specification for reference
|
||||
@@ -883,7 +894,7 @@ proc nvme_init stdcall, pci:dword
|
||||
jb .exit_fail
|
||||
|
||||
; For some reason, bit 7 (No I/O command set supported) is also set to 1 despite bit 0 (NVM command set)
|
||||
; being set to 1.. so I am not sure if bit 7 should be checked at all.. investigate later.
|
||||
; being set to 1.. so I am not sure if bit 7 should be checked at all.. investigate later.
|
||||
mov eax, dword [edi + NVME_MMIO.CAP + 4]
|
||||
test eax, CAP_CSS_NVM_CMDSET
|
||||
jz .exit_fail
|
||||
@@ -891,7 +902,7 @@ proc nvme_init stdcall, pci:dword
|
||||
; Reset controller before we configure it
|
||||
stdcall nvme_controller_reset, esi
|
||||
if __DEBUG__
|
||||
stdcall nvme_wait, edi
|
||||
;stdcall nvme_wait, edi
|
||||
end if
|
||||
mov eax, dword [edi + NVME_MMIO.CAP + 4]
|
||||
and eax, CAP_MPSMIN
|
||||
@@ -949,7 +960,6 @@ proc nvme_init stdcall, pci:dword
|
||||
push ecx
|
||||
stdcall memsetdz, eax, sizeof.CQ_ENTRY * CQ_ENTRIES / 4
|
||||
pop ecx
|
||||
mov dword [edi + ecx + NVM_QUEUE_ENTRY.phase_tag], CQ_PHASE_TAG
|
||||
add ecx, sizeof.NVM_QUEUE_ENTRY
|
||||
cmp ecx, (LAST_QUEUE_ID + 1) * sizeof.NVM_QUEUE_ENTRY
|
||||
jne @b
|
||||
@@ -986,17 +996,19 @@ proc nvme_init stdcall, pci:dword
|
||||
invoke GetPhysAddr
|
||||
; pci:dword, nsid:dword, dptr:dword, cns:byte
|
||||
stdcall nvme_identify, [pci], 0, eax, CNS_IDCS
|
||||
test eax, eax
|
||||
jz .exit_fail
|
||||
mov eax, dword [edi + IDENTC.nn]
|
||||
mov dword [esi + pcidev.nn], eax
|
||||
;DEBUGF DBG_INFO, "nvme%u: Namespace Count: %u\n", [esi + pcidev.num], eax
|
||||
DEBUGF DBG_INFO, "nvme%u: Namespace Count: %u\n", [esi + pcidev.num], eax
|
||||
lea ebx, byte [edi + IDENTC.sn]
|
||||
lea eax, byte [esi + pcidev.serial]
|
||||
stdcall memcpy, eax, ebx, 20
|
||||
;DEBUGF DBG_INFO, "nvme%u: Serial Number: %s\n", [esi + pcidev.num], eax
|
||||
DEBUGF DBG_INFO, "nvme%u: Serial Number: %s\n", [esi + pcidev.num], eax
|
||||
add ebx, 20
|
||||
lea eax, byte [esi + pcidev.model]
|
||||
stdcall memcpy, eax, ebx, 40
|
||||
;DEBUGF DBG_INFO, "nvme%u: Model: %s\n", [esi + pcidev.num], eax
|
||||
DEBUGF DBG_INFO, "nvme%u: Model: %s\n", [esi + pcidev.num], eax
|
||||
mov edx, dword [esi + pcidev.version]
|
||||
|
||||
cmp edx, VS140
|
||||
@@ -1020,7 +1032,11 @@ proc nvme_init stdcall, pci:dword
|
||||
invoke KernelFree, edi
|
||||
|
||||
mov eax, 1 or (1 shl 16) ; CDW11 (set the number of queues we want)
|
||||
LOCK_SPINLOCK
|
||||
stdcall set_features, [pci], NULLPTR, FID_NUMBER_OF_QUEUES, eax
|
||||
call nvme_poll
|
||||
test eax, eax
|
||||
jz .exit_fail
|
||||
mov esi, [pci]
|
||||
mov esi, dword [esi + pcidev.queue_entries]
|
||||
mov esi, dword [esi + NVM_QUEUE_ENTRY.cq_ptr]
|
||||
@@ -1043,25 +1059,21 @@ proc nvme_init stdcall, pci:dword
|
||||
mov eax, dword [esi + NVM_QUEUE_ENTRY.cq_ptr]
|
||||
invoke GetPhysAddr
|
||||
stdcall create_io_completion_queue, [pci], eax, 1, IEN_ON
|
||||
test eax, eax
|
||||
jz .exit_fail
|
||||
;DEBUGF DBG_INFO, "nvme%u: Successfully created I/O completion queue 1\n", [edi + pcidev.num]
|
||||
mov eax, dword [esi + NVM_QUEUE_ENTRY.sq_ptr]
|
||||
invoke GetPhysAddr
|
||||
stdcall create_io_submission_queue, [pci], eax, 1, 1
|
||||
jz .exit_fail
|
||||
;DEBUGF DBG_INFO, "nvme%u: Successfully created I/O submission queue 1\n", [edi + pcidev.num]
|
||||
|
||||
if 1
|
||||
stdcall determine_active_nsids, [pci]
|
||||
test eax, eax
|
||||
jz .exit_fail ; No active NSIDS
|
||||
mov esi, [pci]
|
||||
mov dword [esi + pcidev.nsid], eax
|
||||
;DEBUGF DBG_INFO, "nvme%u: Found active NSID: %u\n", [esi + pcidev.num], eax
|
||||
else
|
||||
mov esi, [pci]
|
||||
xor eax, eax
|
||||
inc eax
|
||||
mov dword [esi + pcidev.nsid], eax
|
||||
end if
|
||||
stdcall determine_active_nsids, [pci]
|
||||
test eax, eax
|
||||
jz .exit_fail ; No active NSIDS
|
||||
mov esi, [pci]
|
||||
mov dword [esi + pcidev.nsid], eax
|
||||
DEBUGF DBG_INFO, "nvme%u: Found active NSID: %u\n", [esi + pcidev.num], eax
|
||||
|
||||
invoke KernelAlloc, 0x1000
|
||||
test eax, eax
|
||||
@@ -1069,6 +1081,8 @@ proc nvme_init stdcall, pci:dword
|
||||
mov edi, eax
|
||||
invoke GetPhysAddr
|
||||
stdcall nvme_identify, [pci], [esi + pcidev.nsid], eax, CNS_IDNS
|
||||
test eax, eax
|
||||
jz .exit_fail
|
||||
invoke KernelAlloc, sizeof.NSINFO
|
||||
test eax, eax
|
||||
jz .exit_fail
|
||||
@@ -1109,7 +1123,11 @@ proc nvme_init stdcall, pci:dword
|
||||
mov edx, NVM_CMD_READ
|
||||
mov dword [eax], 25
|
||||
add edi, 0x5
|
||||
LOCK_SPINLOCK
|
||||
stdcall nvme_readwrite, [esi + pcidev.nsinfo], edi, 0x1000, 0, eax
|
||||
call nvme_poll
|
||||
test eax, eax
|
||||
jz .exit_fail
|
||||
DEBUGF DBG_INFO, "STRING: %s\n", edi
|
||||
add edi, 0x2000
|
||||
DEBUGF DBG_INFO, "STRING: %s\n", edi
|
||||
@@ -1195,6 +1213,32 @@ proc nvme_wait stdcall, mmio:dword
|
||||
|
||||
endp
|
||||
|
||||
proc nvme_poll
|
||||
|
||||
xor ecx, ecx
|
||||
|
||||
@@:
|
||||
inc ecx
|
||||
cmp ecx, 0xffffffff
|
||||
xor eax, eax
|
||||
inc eax
|
||||
xchg eax, dword [spinlock]
|
||||
test eax, eax
|
||||
jnz @b
|
||||
|
||||
; lock was released, return 1
|
||||
xor eax, eax
|
||||
inc eax
|
||||
ret
|
||||
|
||||
@@:
|
||||
; timeout: lock wasn't released, return 0
|
||||
xor eax, eax
|
||||
ret
|
||||
|
||||
endp
|
||||
|
||||
|
||||
; Writes to completion queue 'y' head doorbell
|
||||
proc cqyhdbl_write stdcall, pci:dword, y:dword, cqh:dword
|
||||
|
||||
@@ -1216,7 +1260,7 @@ proc cqyhdbl_write stdcall, pci:dword, y:dword, cqh:dword
|
||||
lea edi, dword [edi + ecx]
|
||||
mov esi, dword [esi + pcidev.io_addr]
|
||||
mov eax, [cqh]
|
||||
;DEBUGF DBG_INFO, "(NVMe) Writing to completion queue doorbell register 0x%x: %u\n", dx, ax
|
||||
;DEBUGF DBG_INFO, "Writing to completion queue doorbell register: %u\n", ax
|
||||
mov word [esi + edx], ax ; Write to CQyHDBL
|
||||
mov word [edi + NVM_QUEUE_ENTRY.head], ax
|
||||
pop edi esi
|
||||
@@ -1246,7 +1290,7 @@ proc sqytdbl_write stdcall, pci:dword, y:word, cmd:dword
|
||||
movzx ecx, [y]
|
||||
imul ecx, sizeof.NVM_QUEUE_ENTRY
|
||||
movzx eax, word [edi + ecx + NVM_QUEUE_ENTRY.tail]
|
||||
cmp ax, NVM_ASQS
|
||||
cmp ax, (NVM_ASQS - 1)
|
||||
jb @f
|
||||
xor ax, ax
|
||||
|
||||
@@ -1268,42 +1312,11 @@ proc sqytdbl_write stdcall, pci:dword, y:word, cmd:dword
|
||||
mov word [edi + ecx + NVM_QUEUE_ENTRY.tail], ax
|
||||
dec ax
|
||||
movzx ecx, [y]
|
||||
stdcall nvme_cmd_wait, [pci], ecx, eax
|
||||
pop edi esi ebx
|
||||
ret
|
||||
|
||||
endp
|
||||
|
||||
proc nvme_cmd_wait stdcall, pci:dword, y:dword, cid:word
|
||||
|
||||
push esi
|
||||
mov esi, [pci]
|
||||
movzx ecx, word [cid]
|
||||
mov edx, [y]
|
||||
imul edx, sizeof.NVM_QUEUE_ENTRY
|
||||
mov esi, dword [esi + pcidev.queue_entries]
|
||||
lea esi, [esi + edx]
|
||||
imul ecx, sizeof.CQ_ENTRY
|
||||
mov eax, dword [esi + NVM_QUEUE_ENTRY.phase_tag]
|
||||
mov esi, dword [esi + NVM_QUEUE_ENTRY.cq_ptr]
|
||||
test eax, CQ_PHASE_TAG
|
||||
jnz .phase_tag_1
|
||||
|
||||
@@:
|
||||
test byte [esi + ecx + CQ_ENTRY.status], CQ_PHASE_TAG
|
||||
jnz @b
|
||||
pop esi
|
||||
ret
|
||||
|
||||
.phase_tag_1:
|
||||
;DEBUGF DBG_INFO, "status: %x\n", [esi + ecx + CQ_ENTRY.status]
|
||||
test byte [esi + ecx + CQ_ENTRY.status], CQ_PHASE_TAG
|
||||
jz .phase_tag_1
|
||||
pop esi
|
||||
ret
|
||||
|
||||
endp
|
||||
|
||||
proc is_queue_full stdcall, tail:word, head:word
|
||||
|
||||
push bx
|
||||
@@ -1315,6 +1328,7 @@ proc is_queue_full stdcall, tail:word, head:word
|
||||
jnz @f
|
||||
cmp ax, NVM_ASQS
|
||||
jne @f
|
||||
pop bx
|
||||
xor eax, eax
|
||||
inc eax
|
||||
ret
|
||||
@@ -1325,6 +1339,7 @@ proc is_queue_full stdcall, tail:word, head:word
|
||||
sub ax, bx
|
||||
cmp ax, 1
|
||||
jne .not_full
|
||||
pop bx
|
||||
xor eax, eax
|
||||
inc eax
|
||||
ret
|
||||
@@ -1351,6 +1366,10 @@ proc consume_cq_entries stdcall, pci:dword, queue:dword
|
||||
test eax, eax
|
||||
jnz .end
|
||||
movzx ecx, word [esi + NVM_QUEUE_ENTRY.head]
|
||||
cmp ecx, (NVM_ACQS - 1)
|
||||
jb .loop
|
||||
xor ecx, ecx
|
||||
mov word [esi + NVM_QUEUE_ENTRY.head], cx
|
||||
|
||||
.loop:
|
||||
cmp cx, word [esi + NVM_QUEUE_ENTRY.tail]
|
||||
@@ -1377,28 +1396,28 @@ proc irq_handler
|
||||
push esi edi
|
||||
mov esi, dword [p_nvme_devices]
|
||||
|
||||
; check if the NVMe device generated an interrupt
|
||||
; check if a NVMe device generated an interrupt
|
||||
invoke PciRead16, dword [esi + pcidev.bus], dword [esi + pcidev.devfn], PCI_header00.status
|
||||
test al, 1000b ; check interrupt status
|
||||
jz .not_our_irq
|
||||
|
||||
mov edi, dword [esi + pcidev.io_addr]
|
||||
mov dword [edi + NVME_MMIO.INTMS], 0x3
|
||||
xor ecx, ecx
|
||||
|
||||
mov eax, dword [spinlock]
|
||||
test eax, eax
|
||||
jz @f ; not locked, so it must be an I/O command
|
||||
stdcall consume_cq_entries, [p_nvme_devices], 0
|
||||
|
||||
@@:
|
||||
push ecx
|
||||
stdcall consume_cq_entries, [p_nvme_devices], ecx
|
||||
pop ecx
|
||||
inc ecx
|
||||
cmp ecx, LAST_QUEUE_ID
|
||||
jng @b
|
||||
stdcall consume_cq_entries, [p_nvme_devices], 1
|
||||
|
||||
; Interrupt handled by driver, return 1
|
||||
mov dword [edi + NVME_MMIO.INTMC], 0x3
|
||||
pop edi esi
|
||||
xor eax, eax
|
||||
inc eax
|
||||
xchg eax, dword [spinlock] ; unlock spinlock
|
||||
mov eax, 1
|
||||
ret
|
||||
|
||||
.not_our_irq:
|
||||
@@ -1434,6 +1453,7 @@ endp
|
||||
align 4
|
||||
p_nvme_devices dd 0
|
||||
pcidevs_len dd 0
|
||||
spinlock dd 0
|
||||
my_service db "NVMe",0 ;max 16 chars include zero
|
||||
disk_functions:
|
||||
dd disk_functions.end - disk_functions
|
||||
|
||||
Reference in New Issue
Block a user