main
1package rfb
2
3import (
4 "context"
5 "crypto/des"
6 "fmt"
7 "strings"
8
9 "goVNC/pkg/transport"
10)
11
12const (
13 // RFBVersion is the RFB protocol version we support.
14 RFBVersion = "RFB 003.008\n"
15)
16
17// SecurityType represents an RFB security type.
18type SecurityType uint8
19
20const (
21 SecurityTypeInvalid SecurityType = 0
22 SecurityTypeNone SecurityType = 1
23 SecurityTypeVNCAuth SecurityType = 2
24 SecurityTypeTight SecurityType = 16
25 SecurityTypeVeNCrypt SecurityType = 19
26)
27
28// HandshakeConfig configures the RFB handshake.
29type HandshakeConfig struct {
30 // SharedSession requests a shared session if true.
31 SharedSession bool
32
33 // Password for VNC authentication (SecurityTypeVNCAuth).
34 Password string
35
36 // AllowedSecurityTypes lists allowed security types.
37 // If empty, only SecurityTypeNone is allowed.
38 AllowedSecurityTypes []SecurityType
39}
40
41// DefaultHandshakeConfig returns default handshake configuration.
42func DefaultHandshakeConfig() *HandshakeConfig {
43 return &HandshakeConfig{
44 SharedSession: true,
45 AllowedSecurityTypes: []SecurityType{SecurityTypeNone},
46 }
47}
48
49// Handshake performs the RFB protocol handshake and returns a Session.
50func Handshake(ctx context.Context, t transport.Transport, cfg *HandshakeConfig) (*Session, error) {
51 if cfg == nil {
52 cfg = DefaultHandshakeConfig()
53 }
54
55 // Step 1: Read server version
56 serverVersionData, err := t.Read(ctx)
57 if err != nil {
58 return nil, fmt.Errorf("reading server version: %w", err)
59 }
60 serverVersion := strings.TrimSpace(string(serverVersionData))
61
62 // Step 2: Send client version
63 err = t.Write(ctx, []byte(RFBVersion))
64 if err != nil {
65 return nil, fmt.Errorf("sending client version: %w", err)
66 }
67
68 // Step 3: Read security types
69 secTypes, err := t.Read(ctx)
70 if err != nil {
71 return nil, fmt.Errorf("reading security types: %w", err)
72 }
73
74 if len(secTypes) < 1 {
75 return nil, fmt.Errorf("empty security types message")
76 }
77
78 // Parse security types
79 numSecTypes := int(secTypes[0])
80 if numSecTypes == 0 {
81 // Version 3.3 error or failure
82 if len(secTypes) >= 5 {
83 errLen := int(secTypes[1])<<24 | int(secTypes[2])<<16 | int(secTypes[3])<<8 | int(secTypes[4])
84 if len(secTypes) >= 5+errLen {
85 return nil, fmt.Errorf("server rejected: %s", string(secTypes[5:5+errLen]))
86 }
87 }
88 return nil, fmt.Errorf("server rejected connection")
89 }
90
91 if len(secTypes) < 1+numSecTypes {
92 return nil, fmt.Errorf("security types message too short")
93 }
94
95 // Find a supported security type
96 var selectedSecurity SecurityType
97 for i := 0; i < numSecTypes; i++ {
98 offered := SecurityType(secTypes[1+i])
99 for _, allowed := range cfg.AllowedSecurityTypes {
100 if offered == allowed {
101 selectedSecurity = offered
102 break
103 }
104 }
105 if selectedSecurity != 0 {
106 break
107 }
108 }
109
110 if selectedSecurity == 0 {
111 return nil, fmt.Errorf("no supported security type (server offered: %v)", secTypes[1:1+numSecTypes])
112 }
113
114 // Step 4: Send security choice
115 err = t.Write(ctx, []byte{byte(selectedSecurity)})
116 if err != nil {
117 return nil, fmt.Errorf("sending security choice: %w", err)
118 }
119
120 // Step 5: Handle security type specific authentication
121 switch selectedSecurity {
122 case SecurityTypeNone:
123 // No authentication required
124 case SecurityTypeVNCAuth:
125 err = handleVNCAuth(ctx, t, cfg.Password)
126 if err != nil {
127 return nil, fmt.Errorf("VNC authentication: %w", err)
128 }
129 default:
130 return nil, fmt.Errorf("unsupported security type: %d", selectedSecurity)
131 }
132
133 // Step 6: Read security result
134 secResult, err := t.Read(ctx)
135 if err != nil {
136 return nil, fmt.Errorf("reading security result: %w", err)
137 }
138
139 if len(secResult) < 4 {
140 // Some servers don't send security result for None auth in older versions
141 if selectedSecurity != SecurityTypeNone {
142 return nil, fmt.Errorf("security result too short: %d bytes", len(secResult))
143 }
144 // Treat as success for None auth
145 } else {
146 result := uint32(secResult[0])<<24 | uint32(secResult[1])<<16 | uint32(secResult[2])<<8 | uint32(secResult[3])
147 if result != 0 {
148 return nil, fmt.Errorf("security handshake failed: result=%d", result)
149 }
150 }
151
152 // Step 7: Send ClientInit
153 clientInit := byte(0)
154 if cfg.SharedSession {
155 clientInit = 1
156 }
157 err = t.Write(ctx, []byte{clientInit})
158 if err != nil {
159 return nil, fmt.Errorf("sending client init: %w", err)
160 }
161
162 // Step 8: Read ServerInit
163 serverInit, err := t.Read(ctx)
164 if err != nil {
165 return nil, fmt.Errorf("reading server init: %w", err)
166 }
167
168 session, err := ParseServerInit(serverInit)
169 if err != nil {
170 return nil, fmt.Errorf("parsing server init: %w", err)
171 }
172 session.ServerVersion = serverVersion
173
174 return session, nil
175}
176
177// handleVNCAuth handles VNC authentication (DES challenge-response).
178func handleVNCAuth(ctx context.Context, t transport.Transport, password string) error {
179 // Read 16-byte challenge
180 challenge, err := t.Read(ctx)
181 if err != nil {
182 return fmt.Errorf("reading challenge: %w", err)
183 }
184 if len(challenge) != 16 {
185 return fmt.Errorf("unexpected challenge length: %d", len(challenge))
186 }
187
188 // Encrypt challenge with password using DES
189 response := vncAuthEncrypt(challenge, password)
190
191 // Send 16-byte response
192 err = t.Write(ctx, response)
193 if err != nil {
194 return fmt.Errorf("sending response: %w", err)
195 }
196
197 return nil
198}
199
200// vncAuthEncrypt encrypts the VNC authentication challenge.
201// VNC uses a weird DES variant where key bits are reversed.
202func vncAuthEncrypt(challenge []byte, password string) []byte {
203 // Pad or truncate password to 8 bytes
204 key := make([]byte, 8)
205 copy(key, password)
206
207 // Reverse bits in each byte of the key (VNC quirk)
208 for i := range key {
209 key[i] = reverseBits(key[i])
210 }
211
212 // Create DES cipher
213 cipher, err := des.NewCipher(key)
214 if err != nil {
215 // Should not happen with 8-byte key
216 return challenge
217 }
218
219 // Encrypt with DES in ECB mode (2 blocks of 8 bytes)
220 response := make([]byte, 16)
221 cipher.Encrypt(response[0:8], challenge[0:8])
222 cipher.Encrypt(response[8:16], challenge[8:16])
223
224 return response
225}
226
227// reverseBits reverses the bits in a byte.
228// VNC authentication requires this quirk for the DES key.
229func reverseBits(b byte) byte {
230 var result byte
231 for i := 0; i < 8; i++ {
232 if b&(1<<i) != 0 {
233 result |= 1 << (7 - i)
234 }
235 }
236 return result
237}