Entering protected mode from a DOS program

Question

I think i've read a dozen or so questions that are basically a duplicate of this one, but I still haven't found a solution.

The desired result is to enter protected mode and halt with no faults. The problem i'm experiencing is a triple fault after executing the intersegment jmp with 6 byte immediate value.

here's my code that produces the fault in DOSBox and on a Pentium II PC running MS-DOS 7. Assembler is MASM 5.10

single segment stack                                                      
assume cs:single,ds:single 

gdt        dq 0

c_limit_lo dw 0ffffh     
c_base_lo  dw 0
c_base_mid db 0
c_priv     db 10011110b ;present set, highest priv, type set, conforming, read      
c_limit_hi db 11001111b ;granularity set, operand size 32  
c_base_hi  db 0

d_limit_lo dw 0ffffh
d_base_lo  dw 0
d_base_mid db 0
d_priv     db 10010010b ;present set, highest priv, type clr, expand dn, write  
d_limit_hi db 11001111b ;granularity set, big set 
d_base_hi  db 0
gdt_end:

gdt_limit  dw gdt_end-offset gdt-1
gdt_addr   dd ?

start:
     mov ax, cs
     mov ds, ax
                                   ;calc phys address of current code segment and
                                   ;insert it into code and data descriptors 
     .386p                         
     xor eax, eax
     mov ax, cs                    
     mov cl, 4              
     shl eax, cl                   ;multiply cs by 16 to get phys address of seg
     mov edx, eax
     mov c_base_lo, ax
     mov d_base_lo, ax             ;low word
     mov cl, 16
     shr eax, cl
     mov c_base_mid, al
     mov d_base_mid, al            ;middle byte
     mov c_base_hi, ah
     mov d_base_hi, ah             ;high byte
     add edx, offset gdt           ;add offset of gdt 
     mov gdt_addr, edx             ;gdt address set 

                                   ;attempt to enter protected mode
     cli                           ;disable interrupts
     in al, 70h
     or al, 80h
     out 70h, al                   ;turn off nonmasked interrupts
     in al, 92h
     or al, 2
     out 92h, al                   ;enable A20 line
     lgdt [gdt_limit]
     mov eax, cr0         
     or eax, 1
     mov cr0, eax                  ;enter protected mode
     db 66h                        ;specify 32-bit operand
     jmp_op  db 0eah               ;manually encoded "jmp 8h:enter_32" TRIPLE FAULT
     jmp_loc_lo dw offset enter_32
     jmp_loc_hi dw 0
     jmp_sel dw 8
enter_32:  
     mov eax, 0ffffffffh           ;sometimes doesn't triple fault on infinite jump or hlt instruction
back:jmp back                      ;but always triple faults on mov



the_stack db 64 dup (0ffh)          ;64 byte stack

single ends                                                                  
end start

The triple fault seems to be "luck" based to some extent. Certain configurations of 0x67 prefixes and nops after the far jump cause the cpu to behave as if it's halted. I don't really understand it.

I think i'm generating the wrong jump target.

Update: It's not faulting with single byte instructions (instructions with a single encoding regardless of cpu mode). I think I'll try jumping into a USE32 defined segment.

This code does not fault:

     jmp_op  db 0eah               
     jmp_loc_lo dw offset enter_32
     jmp_loc_hi dw 0
     jmp_sel dw 8
enter_32:  
     aaa
     daa
     cmc
     cld
     cli
     stc
     nop
     aaa
     daa
     cmc
     cld
     cli
     stc
     nop
     hlt

user_5407947 · Answer

Answered by fuz, I just needed to jump into code that was assembled for 32-bit mode. You tell the assembler to make 32-bit code by defining a segment with the USE32 keyword.

Complete protected mode program with a VGA demo and a mode switch back to real mode:

single segment stack                                                      
assume cs:single,ds:single 

gdt        dq 0                             ;global descriptor table
p_code     dq 00cf9e000000ffffh             ;protected mode code descriptor
p_data     dq 00cf92000000ffffh             ;protected mode data descriptor
r_code     dq 008f9a000000ffffh             ;real mode code descriptor
r_data     dq 008f92000000ffffh             ;real mode data descriptor
v_buff     dq 00cf920a0000ffffh             ;vga buffer descriptor

gdt_limit  dw offset gdt_limit-offset gdt-1 ;gdt_limit <- gdt byte size -1
gdt_addr   dd offset gdt                    ;gdt_addr <- offset of gdt, phys address of
                                            ;code segment will be added
start:
    mov ax, cs 
    mov ds, ax                   ;ds = cs, single segment
    mov ax, 13h
    int 10h                      ;enter vga 320x200x256

    .386p                        ;enable 32-bit extensions
    xor eax, eax                 ;clear high word of eax
    mov ax, cs                   ;eax <- cs 
    shl eax, 4                   ;eax <- physical address of cs 
    add [gdt_addr], eax          ;gdt_addr <- physical address of gdt
    mov word ptr [r_code+2], ax  
    mov word ptr [r_data+2], ax  ;insert low word of cs phys address
    shr eax, 16
    mov byte ptr [r_code+4], al
    mov byte ptr [r_data+4], al  ;insert middle byte of cs address
    mov byte ptr [r_code+7], ah
    mov byte ptr [r_data+7], ah  ;insert high byte of cs address

    xor eax, eax                 ;clear high word of eax
    mov ax, seg32                ;eax <- seg32 segment address
    shl eax, 4                   ;eax <- physical address of seg32 
    mov word ptr [p_code+2], ax   
    mov word ptr [p_data+2], ax  ;insert low word of seg32 phys address
    shr eax, 16
    mov byte ptr [p_code+4], al
    mov byte ptr [p_data+4], al  ;insert middle byte of seg32 address
    mov byte ptr [p_code+7], ah
    mov byte ptr [p_data+7], ah  ;insert high byte of seg32 address

    cli                          ;disable interrupts
    in al, 70h                   ;al <- cmos ram index register port
    or al, 80h                   ;set bit 7 to disable nmi 
    out 70h, al                  ;nmi disabled
    in al, 92h                   ;al <- ps/2 system control port
    or al, 2                     ;set bit 1 to enable a20
    out 92h, al                  ;a20 enabled

    lgdt [gdt_limit]             ;load gdt 
    mov eax, cr0                  
    or eax, 1                    ;set pe bit
    mov cr0, eax                 ;enter protected mode
    db 66h                       ;specify 32-bit operand
    db 0eah                      ;manually encoded jmp 8h:0, jump to offset 0 of seg32
    dd offset enter_32                  
    dw 8 

ret_real:
    mov eax, cr0
    and al, 11111110b            ;clear pe bit
    mov cr0, eax                 ;real mode enabled     
    db 0eah                      ;jmp single:real_cs to load cs:ip
    dw offset real_cs
    dw seg single  

real_cs:
    mov ax, cs                      
    mov ds, ax                   ;ds = cs
    mov ss, ax                   ;ss = cs
    mov sp, offset s16_end       ;top of stack is end of stack
    in al, 70h                   ;al <- cmos ram index register port
    and al, 01111111b            ;clear bit 7 to enable nmi 
    out 70h, al                  ;nmi enabled
    sti                          ;enable interrupts

    mov ax, 40h
    mov es, ax                   ;access kbd data area via segment 40h
    mov word ptr es:[1ah], 1eh   ;set the kbd buff head to start of buff
    mov word ptr es:[1ch], 1eh   ;set kbd buff tail to same as buff head
                                 ;now the keyboard buffer is cleared.
    xor ah, ah                   ;select video mode function
    mov al, 3                    ;select 80x25 16 colors
    int 10h                      ;restore vga compatible text mode
    mov ax, 4c00h                ;Terminate process function selected
    int 21h                      ;return to ms-dos

s16 db 256 dup (0ffh)            ;needed 256 bytes to call int 10h on fx5200 vga bios
s16_end: 
single ends   


seg32 segment use32
assume cs:seg32,ds:seg32

enter_32:
    mov ax, 10h                  ;protected mode data segment selector
    mov ds, ax                   ;ds references main data segment
    mov ss, ax                   ;stack is in main data segment 
    mov esp, offset s32_end      ;initial top of stack is end of stack
    mov ax, 28h                  ;vga buffer selector
    mov es, ax                   ;es references vga buffer  
    mov eax, 0ffffffffh          ;initialize eax
write_scr:
    inc al
    inc ah
    rol eax, 16
    inc al
    inc ah                       ;increment each byte of eax
    xor edi, edi                 ;init index
    mov ecx, 320*200/4           ;vga buffer length in bytes

    push eax
    mov dx, 3dah                 ;dx <- vga status register
vrb_set:
    in al, dx                    ;al <- status byte
    test al, 8                   ;is bit vertical retrace bit set
    jnz vrb_set                  ;if so, wait for it to clear
vrb_clr:                         ;when clear, wait for it to be set
    in al, dx
    test al, 8
    jz vrb_clr                   ;loop back until vertical retrace bit has been set
    pop eax
    rep stosd                    ;fill vga buffer

    push eax
    in al, 60h                   ;al <- keyboard data port
    mov ebx, eax                 
    pop eax                     
    cmp bl, 1                    ;escape key scancode? 
    jne write_scr                ;if not, update screen
    mov ax, 20h                  ;real mode data selector
    mov ds, ax                   
    mov es, ax                   ;setup ds and es for real mode
    db 0eah                      ;jmp 18h:ret_real to load real mode code descriptor 
    dd offset ret_real
    dw 18h

s32 db 128 dup (0ffh)            ;128 byte stack
s32_end:
seg32 ends

end start

Assembling code for 16-bit real mode and running it in 32-bit protected mode may produce unexpected behaviour and crashes. This 16-bit code:

    mov eax, 0ffffffffh
back:jmp back

Is encoded as:

66B8FFFFFFFF      mov eax,0xffffffff
EBFE              jmp short 0x6

However, if this sequence of bytes is decoded as 32-bit protected mode instructions they would be interpreted as:

66B8FFFF          mov ax,0xffff
FF                db 0xff
FF                db 0xff
EBFE              jmp short 0x6

After moving 0xffff to register AX the processor would raise a general protection fault (#GP) when it found the invalid instruction (byte 0xff). In the absence of a proper Interrupt Descriptor Table (IDT) and exception handler for #GP a triple fault occurs.

Entering protected mode from a DOS program

1 Answers