1 //========================================================================
5 // Copyright 1996-2003 Glyph & Cog, LLC
7 //========================================================================
11 #ifdef USE_GCC_PRAGMAS
12 #pragma implementation
19 static void rc4InitKey(Guchar
*key
, int keyLen
, Guchar
*state
);
20 static Guchar
rc4DecryptByte(Guchar
*state
, Guchar
*x
, Guchar
*y
, Guchar c
);
21 static void md5(Guchar
*msg
, int msgLen
, Guchar
*digest
);
23 static Guchar passwordPad
[32] = {
24 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
25 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
26 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
27 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
30 //------------------------------------------------------------------------
32 //------------------------------------------------------------------------
34 Decrypt::Decrypt(Guchar
*fileKey
, int keyLength
, int objNum
, int objGen
) {
37 // construct object key
38 for (i
= 0; i
< keyLength
; ++i
) {
39 objKey
[i
] = fileKey
[i
];
41 objKey
[keyLength
] = objNum
& 0xff;
42 objKey
[keyLength
+ 1] = (objNum
>> 8) & 0xff;
43 objKey
[keyLength
+ 2] = (objNum
>> 16) & 0xff;
44 objKey
[keyLength
+ 3] = objGen
& 0xff;
45 objKey
[keyLength
+ 4] = (objGen
>> 8) & 0xff;
46 md5(objKey
, keyLength
+ 5, objKey
);
48 // set up for decryption
50 if ((objKeyLength
= keyLength
+ 5) > 16) {
53 rc4InitKey(objKey
, objKeyLength
, state
);
56 void Decrypt::reset() {
58 rc4InitKey(objKey
, objKeyLength
, state
);
61 Guchar
Decrypt::decryptByte(Guchar c
) {
62 return rc4DecryptByte(state
, &x
, &y
, c
);
65 GBool
Decrypt::makeFileKey(int encVersion
, int encRevision
, int keyLength
,
66 GooString
*ownerKey
, GooString
*userKey
,
67 int permissions
, GooString
*fileID
,
68 GooString
*ownerPassword
, GooString
*userPassword
,
69 Guchar
*fileKey
, GBool encryptMetadata
,
70 GBool
*ownerPasswordOk
) {
71 Guchar test
[32], test2
[32];
72 GooString
*userPassword2
;
78 // try using the supplied owner password to generate the user password
79 *ownerPasswordOk
= gFalse
;
81 len
= ownerPassword
->getLength();
83 memcpy(test
, ownerPassword
->getCString(), len
);
84 memcpy(test
+ len
, passwordPad
, 32 - len
);
86 memcpy(test
, ownerPassword
->getCString(), 32);
89 if (encRevision
== 3) {
90 for (i
= 0; i
< 50; ++i
) {
94 if (encRevision
== 2) {
95 rc4InitKey(test
, keyLength
, fState
);
97 for (i
= 0; i
< 32; ++i
) {
98 test2
[i
] = rc4DecryptByte(fState
, &fx
, &fy
, ownerKey
->getChar(i
));
101 memcpy(test2
, ownerKey
->getCString(), 32);
102 for (i
= 19; i
>= 0; --i
) {
103 for (j
= 0; j
< keyLength
; ++j
) {
104 tmpKey
[j
] = test
[j
] ^ i
;
106 rc4InitKey(tmpKey
, keyLength
, fState
);
108 for (j
= 0; j
< 32; ++j
) {
109 test2
[j
] = rc4DecryptByte(fState
, &fx
, &fy
, test2
[j
]);
113 userPassword2
= new GooString((char *)test2
, 32);
114 if (makeFileKey2(encVersion
, encRevision
, keyLength
, ownerKey
, userKey
,
115 permissions
, fileID
, userPassword2
, fileKey
,
117 *ownerPasswordOk
= gTrue
;
118 delete userPassword2
;
121 delete userPassword2
;
124 // try using the supplied user password
125 return makeFileKey2(encVersion
, encRevision
, keyLength
, ownerKey
, userKey
,
126 permissions
, fileID
, userPassword
, fileKey
,
130 GBool
Decrypt::makeFileKey2(int encVersion
, int encRevision
, int keyLength
,
131 GooString
*ownerKey
, GooString
*userKey
,
132 int permissions
, GooString
*fileID
,
133 GooString
*userPassword
, Guchar
*fileKey
,
134 GBool encryptMetadata
) {
144 buf
= (Guchar
*)gmalloc(72 + fileID
->getLength());
146 len
= userPassword
->getLength();
148 memcpy(buf
, userPassword
->getCString(), len
);
149 memcpy(buf
+ len
, passwordPad
, 32 - len
);
151 memcpy(buf
, userPassword
->getCString(), 32);
154 memcpy(buf
, passwordPad
, 32);
156 memcpy(buf
+ 32, ownerKey
->getCString(), 32);
157 buf
[64] = permissions
& 0xff;
158 buf
[65] = (permissions
>> 8) & 0xff;
159 buf
[66] = (permissions
>> 16) & 0xff;
160 buf
[67] = (permissions
>> 24) & 0xff;
161 memcpy(buf
+ 68, fileID
->getCString(), fileID
->getLength());
162 len
= 68 + fileID
->getLength();
163 if (!encryptMetadata
) {
169 md5(buf
, len
, fileKey
);
170 if (encRevision
== 3) {
171 for (i
= 0; i
< 50; ++i
) {
172 md5(fileKey
, keyLength
, fileKey
);
176 // test user password
177 if (encRevision
== 2) {
178 rc4InitKey(fileKey
, keyLength
, fState
);
180 for (i
= 0; i
< 32; ++i
) {
181 test
[i
] = rc4DecryptByte(fState
, &fx
, &fy
, userKey
->getChar(i
));
183 ok
= memcmp(test
, passwordPad
, 32) == 0;
184 } else if (encRevision
== 3) {
185 memcpy(test
, userKey
->getCString(), 32);
186 for (i
= 19; i
>= 0; --i
) {
187 for (j
= 0; j
< keyLength
; ++j
) {
188 tmpKey
[j
] = fileKey
[j
] ^ i
;
190 rc4InitKey(tmpKey
, keyLength
, fState
);
192 for (j
= 0; j
< 32; ++j
) {
193 test
[j
] = rc4DecryptByte(fState
, &fx
, &fy
, test
[j
]);
196 memcpy(buf
, passwordPad
, 32);
197 memcpy(buf
+ 32, fileID
->getCString(), fileID
->getLength());
198 md5(buf
, 32 + fileID
->getLength(), buf
);
199 ok
= memcmp(test
, buf
, 16) == 0;
208 //------------------------------------------------------------------------
209 // RC4-compatible decryption
210 //------------------------------------------------------------------------
212 static void rc4InitKey(Guchar
*key
, int keyLen
, Guchar
*state
) {
213 Guchar index1
, index2
;
217 for (i
= 0; i
< 256; ++i
)
220 for (i
= 0; i
< 256; ++i
) {
221 index2
= (key
[index1
] + state
[i
] + index2
) % 256;
223 state
[i
] = state
[index2
];
225 index1
= (index1
+ 1) % keyLen
;
229 static Guchar
rc4DecryptByte(Guchar
*state
, Guchar
*x
, Guchar
*y
, Guchar c
) {
230 Guchar x1
, y1
, tx
, ty
;
232 x1
= *x
= (*x
+ 1) % 256;
233 y1
= *y
= (state
[*x
] + *y
) % 256;
238 return c
^ state
[(tx
+ ty
) % 256];
241 //------------------------------------------------------------------------
242 // MD5 message digest
243 //------------------------------------------------------------------------
245 // this works around a bug in older Sun compilers
246 static inline Gulong
rotateLeft(Gulong x
, int r
) {
248 return ((x
<< r
) | (x
>> (32 - r
))) & 0xffffffff;
251 static inline Gulong
md5Round1(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
252 Gulong Xk
, Gulong s
, Gulong Ti
) {
253 return b
+ rotateLeft((a
+ ((b
& c
) | (~b
& d
)) + Xk
+ Ti
), s
);
256 static inline Gulong
md5Round2(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
257 Gulong Xk
, Gulong s
, Gulong Ti
) {
258 return b
+ rotateLeft((a
+ ((b
& d
) | (c
& ~d
)) + Xk
+ Ti
), s
);
261 static inline Gulong
md5Round3(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
262 Gulong Xk
, Gulong s
, Gulong Ti
) {
263 return b
+ rotateLeft((a
+ (b
^ c
^ d
) + Xk
+ Ti
), s
);
266 static inline Gulong
md5Round4(Gulong a
, Gulong b
, Gulong c
, Gulong d
,
267 Gulong Xk
, Gulong s
, Gulong Ti
) {
268 return b
+ rotateLeft((a
+ (c
^ (b
| ~d
)) + Xk
+ Ti
), s
);
271 static void md5(Guchar
*msg
, int msgLen
, Guchar
*digest
) {
273 Gulong a
, b
, c
, d
, aa
, bb
, cc
, dd
;
277 // compute number of 64-byte blocks
278 // (length + pad byte (0x80) + 8 bytes for length)
279 n64
= (msgLen
+ 1 + 8 + 63) / 64;
281 // initialize a, b, c, d
287 // loop through blocks
289 for (i
= 0; i
< n64
; ++i
) {
291 // grab a 64-byte block
292 for (j
= 0; j
< 16 && k
< msgLen
- 3; ++j
, k
+= 4)
293 x
[j
] = (((((msg
[k
+3] << 8) + msg
[k
+2]) << 8) + msg
[k
+1]) << 8) + msg
[k
];
296 x
[j
] = 0x80000000 + (((msg
[k
+2] << 8) + msg
[k
+1]) << 8) + msg
[k
];
297 else if (k
== msgLen
- 2)
298 x
[j
] = 0x800000 + (msg
[k
+1] << 8) + msg
[k
];
299 else if (k
== msgLen
- 1)
300 x
[j
] = 0x8000 + msg
[k
];
316 a
= md5Round1(a
, b
, c
, d
, x
[0], 7, 0xd76aa478);
317 d
= md5Round1(d
, a
, b
, c
, x
[1], 12, 0xe8c7b756);
318 c
= md5Round1(c
, d
, a
, b
, x
[2], 17, 0x242070db);
319 b
= md5Round1(b
, c
, d
, a
, x
[3], 22, 0xc1bdceee);
320 a
= md5Round1(a
, b
, c
, d
, x
[4], 7, 0xf57c0faf);
321 d
= md5Round1(d
, a
, b
, c
, x
[5], 12, 0x4787c62a);
322 c
= md5Round1(c
, d
, a
, b
, x
[6], 17, 0xa8304613);
323 b
= md5Round1(b
, c
, d
, a
, x
[7], 22, 0xfd469501);
324 a
= md5Round1(a
, b
, c
, d
, x
[8], 7, 0x698098d8);
325 d
= md5Round1(d
, a
, b
, c
, x
[9], 12, 0x8b44f7af);
326 c
= md5Round1(c
, d
, a
, b
, x
[10], 17, 0xffff5bb1);
327 b
= md5Round1(b
, c
, d
, a
, x
[11], 22, 0x895cd7be);
328 a
= md5Round1(a
, b
, c
, d
, x
[12], 7, 0x6b901122);
329 d
= md5Round1(d
, a
, b
, c
, x
[13], 12, 0xfd987193);
330 c
= md5Round1(c
, d
, a
, b
, x
[14], 17, 0xa679438e);
331 b
= md5Round1(b
, c
, d
, a
, x
[15], 22, 0x49b40821);
334 a
= md5Round2(a
, b
, c
, d
, x
[1], 5, 0xf61e2562);
335 d
= md5Round2(d
, a
, b
, c
, x
[6], 9, 0xc040b340);
336 c
= md5Round2(c
, d
, a
, b
, x
[11], 14, 0x265e5a51);
337 b
= md5Round2(b
, c
, d
, a
, x
[0], 20, 0xe9b6c7aa);
338 a
= md5Round2(a
, b
, c
, d
, x
[5], 5, 0xd62f105d);
339 d
= md5Round2(d
, a
, b
, c
, x
[10], 9, 0x02441453);
340 c
= md5Round2(c
, d
, a
, b
, x
[15], 14, 0xd8a1e681);
341 b
= md5Round2(b
, c
, d
, a
, x
[4], 20, 0xe7d3fbc8);
342 a
= md5Round2(a
, b
, c
, d
, x
[9], 5, 0x21e1cde6);
343 d
= md5Round2(d
, a
, b
, c
, x
[14], 9, 0xc33707d6);
344 c
= md5Round2(c
, d
, a
, b
, x
[3], 14, 0xf4d50d87);
345 b
= md5Round2(b
, c
, d
, a
, x
[8], 20, 0x455a14ed);
346 a
= md5Round2(a
, b
, c
, d
, x
[13], 5, 0xa9e3e905);
347 d
= md5Round2(d
, a
, b
, c
, x
[2], 9, 0xfcefa3f8);
348 c
= md5Round2(c
, d
, a
, b
, x
[7], 14, 0x676f02d9);
349 b
= md5Round2(b
, c
, d
, a
, x
[12], 20, 0x8d2a4c8a);
352 a
= md5Round3(a
, b
, c
, d
, x
[5], 4, 0xfffa3942);
353 d
= md5Round3(d
, a
, b
, c
, x
[8], 11, 0x8771f681);
354 c
= md5Round3(c
, d
, a
, b
, x
[11], 16, 0x6d9d6122);
355 b
= md5Round3(b
, c
, d
, a
, x
[14], 23, 0xfde5380c);
356 a
= md5Round3(a
, b
, c
, d
, x
[1], 4, 0xa4beea44);
357 d
= md5Round3(d
, a
, b
, c
, x
[4], 11, 0x4bdecfa9);
358 c
= md5Round3(c
, d
, a
, b
, x
[7], 16, 0xf6bb4b60);
359 b
= md5Round3(b
, c
, d
, a
, x
[10], 23, 0xbebfbc70);
360 a
= md5Round3(a
, b
, c
, d
, x
[13], 4, 0x289b7ec6);
361 d
= md5Round3(d
, a
, b
, c
, x
[0], 11, 0xeaa127fa);
362 c
= md5Round3(c
, d
, a
, b
, x
[3], 16, 0xd4ef3085);
363 b
= md5Round3(b
, c
, d
, a
, x
[6], 23, 0x04881d05);
364 a
= md5Round3(a
, b
, c
, d
, x
[9], 4, 0xd9d4d039);
365 d
= md5Round3(d
, a
, b
, c
, x
[12], 11, 0xe6db99e5);
366 c
= md5Round3(c
, d
, a
, b
, x
[15], 16, 0x1fa27cf8);
367 b
= md5Round3(b
, c
, d
, a
, x
[2], 23, 0xc4ac5665);
370 a
= md5Round4(a
, b
, c
, d
, x
[0], 6, 0xf4292244);
371 d
= md5Round4(d
, a
, b
, c
, x
[7], 10, 0x432aff97);
372 c
= md5Round4(c
, d
, a
, b
, x
[14], 15, 0xab9423a7);
373 b
= md5Round4(b
, c
, d
, a
, x
[5], 21, 0xfc93a039);
374 a
= md5Round4(a
, b
, c
, d
, x
[12], 6, 0x655b59c3);
375 d
= md5Round4(d
, a
, b
, c
, x
[3], 10, 0x8f0ccc92);
376 c
= md5Round4(c
, d
, a
, b
, x
[10], 15, 0xffeff47d);
377 b
= md5Round4(b
, c
, d
, a
, x
[1], 21, 0x85845dd1);
378 a
= md5Round4(a
, b
, c
, d
, x
[8], 6, 0x6fa87e4f);
379 d
= md5Round4(d
, a
, b
, c
, x
[15], 10, 0xfe2ce6e0);
380 c
= md5Round4(c
, d
, a
, b
, x
[6], 15, 0xa3014314);
381 b
= md5Round4(b
, c
, d
, a
, x
[13], 21, 0x4e0811a1);
382 a
= md5Round4(a
, b
, c
, d
, x
[4], 6, 0xf7537e82);
383 d
= md5Round4(d
, a
, b
, c
, x
[11], 10, 0xbd3af235);
384 c
= md5Round4(c
, d
, a
, b
, x
[2], 15, 0x2ad7d2bb);
385 b
= md5Round4(b
, c
, d
, a
, x
[9], 21, 0xeb86d391);
387 // increment a, b, c, d
394 // break digest into bytes
395 digest
[0] = (Guchar
)(a
& 0xff);
396 digest
[1] = (Guchar
)((a
>>= 8) & 0xff);
397 digest
[2] = (Guchar
)((a
>>= 8) & 0xff);
398 digest
[3] = (Guchar
)((a
>>= 8) & 0xff);
399 digest
[4] = (Guchar
)(b
& 0xff);
400 digest
[5] = (Guchar
)((b
>>= 8) & 0xff);
401 digest
[6] = (Guchar
)((b
>>= 8) & 0xff);
402 digest
[7] = (Guchar
)((b
>>= 8) & 0xff);
403 digest
[8] = (Guchar
)(c
& 0xff);
404 digest
[9] = (Guchar
)((c
>>= 8) & 0xff);
405 digest
[10] = (Guchar
)((c
>>= 8) & 0xff);
406 digest
[11] = (Guchar
)((c
>>= 8) & 0xff);
407 digest
[12] = (Guchar
)(d
& 0xff);
408 digest
[13] = (Guchar
)((d
>>= 8) & 0xff);
409 digest
[14] = (Guchar
)((d
>>= 8) & 0xff);
410 digest
[15] = (Guchar
)((d
>>= 8) & 0xff);