Skip to content

Commit eeeec65

Browse files
jasonpaulosrobdmoore
jasonpaulos
authored and
robdmoore
committed
feat: Expand bigint support to have more configurability
1 parent 992882b commit eeeec65

11 files changed

+953
-183
lines changed

‎README.md

+127-104
Large diffs are not rendered by default.

‎src/Decoder.ts

+29-38
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { prettyByte } from "./utils/prettyByte";
22
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
3-
import { getInt64, getUint64, UINT32_MAX } from "./utils/int";
3+
import { IntMode, getInt64, getUint64, convertSafeIntegerToMode, UINT32_MAX } from "./utils/int";
44
import { utf8Decode } from "./utils/utf8";
55
import { createDataView, ensureUint8Array } from "./utils/typedArrays";
66
import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder";
@@ -16,10 +16,17 @@ export type DecoderOptions<ContextType = undefined> = Readonly<
1616
* Depends on ES2020's {@link DataView#getBigInt64} and
1717
* {@link DataView#getBigUint64}.
1818
*
19-
* Defaults to false.
19+
* Defaults to false. If true, equivalent to intMode: IntMode.AS_ENCODED.
2020
*/
2121
useBigInt64: boolean;
2222

23+
/**
24+
* Allows for more fine-grained control of BigInt handling, overrides useBigInt64.
25+
*
26+
* Defaults to IntMode.AS_ENCODED if useBigInt64 is true or IntMode.UNSAFE_NUMBER otherwise.
27+
*/
28+
intMode?: IntMode;
29+
2330
/**
2431
* Maximum string length.
2532
*
@@ -194,7 +201,7 @@ const sharedCachedKeyDecoder = new CachedKeyDecoder();
194201
export class Decoder<ContextType = undefined> {
195202
private readonly extensionCodec: ExtensionCodecType<ContextType>;
196203
private readonly context: ContextType;
197-
private readonly useBigInt64: boolean;
204+
private readonly intMode: IntMode;
198205
private readonly maxStrLength: number;
199206
private readonly maxBinLength: number;
200207
private readonly maxArrayLength: number;
@@ -214,7 +221,7 @@ export class Decoder<ContextType = undefined> {
214221
this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType<ContextType>);
215222
this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined
216223

217-
this.useBigInt64 = options?.useBigInt64 ?? false;
224+
this.intMode = options?.intMode ?? options?.useBigInt64 ? IntMode.AS_ENCODED : IntMode.UNSAFE_NUMBER;
218225
this.maxStrLength = options?.maxStrLength ?? UINT32_MAX;
219226
this.maxBinLength = options?.maxBinLength ?? UINT32_MAX;
220227
this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX;
@@ -371,11 +378,11 @@ export class Decoder<ContextType = undefined> {
371378

372379
if (headByte >= 0xe0) {
373380
// negative fixint (111x xxxx) 0xe0 - 0xff
374-
object = headByte - 0x100;
381+
object = this.convertNumber(headByte - 0x100);
375382
} else if (headByte < 0xc0) {
376383
if (headByte < 0x80) {
377384
// positive fixint (0xxx xxxx) 0x00 - 0x7f
378-
object = headByte;
385+
object = this.convertNumber(headByte);
379386
} else if (headByte < 0x90) {
380387
// fixmap (1000 xxxx) 0x80 - 0x8f
381388
const size = headByte - 0x80;
@@ -418,36 +425,28 @@ export class Decoder<ContextType = undefined> {
418425
object = this.readF64();
419426
} else if (headByte === 0xcc) {
420427
// uint 8
421-
object = this.readU8();
428+
object = this.convertNumber(this.readU8());
422429
} else if (headByte === 0xcd) {
423430
// uint 16
424-
object = this.readU16();
431+
object = this.convertNumber(this.readU16());
425432
} else if (headByte === 0xce) {
426433
// uint 32
427-
object = this.readU32();
434+
object = this.convertNumber(this.readU32());
428435
} else if (headByte === 0xcf) {
429436
// uint 64
430-
if (this.useBigInt64) {
431-
object = this.readU64AsBigInt();
432-
} else {
433-
object = this.readU64();
434-
}
437+
object = this.readU64();
435438
} else if (headByte === 0xd0) {
436439
// int 8
437-
object = this.readI8();
440+
object = this.convertNumber(this.readI8());
438441
} else if (headByte === 0xd1) {
439442
// int 16
440-
object = this.readI16();
443+
object = this.convertNumber(this.readI16());
441444
} else if (headByte === 0xd2) {
442445
// int 32
443-
object = this.readI32();
446+
object = this.convertNumber(this.readI32());
444447
} else if (headByte === 0xd3) {
445448
// int 64
446-
if (this.useBigInt64) {
447-
object = this.readI64AsBigInt();
448-
} else {
449-
object = this.readI64();
450-
}
449+
object = this.readI64();
451450
} else if (headByte === 0xd9) {
452451
// str 8
453452
const byteLength = this.lookU8();
@@ -692,6 +691,10 @@ export class Decoder<ContextType = undefined> {
692691
return this.extensionCodec.decode(data, extType, this.context);
693692
}
694693

694+
private convertNumber(value: number): number | bigint {
695+
return convertSafeIntegerToMode(value, this.intMode);
696+
}
697+
695698
private lookU8() {
696699
return this.view.getUint8(this.pos);
697700
}
@@ -740,26 +743,14 @@ export class Decoder<ContextType = undefined> {
740743
return value;
741744
}
742745

743-
private readU64(): number {
744-
const value = getUint64(this.view, this.pos);
745-
this.pos += 8;
746-
return value;
747-
}
748-
749-
private readI64(): number {
750-
const value = getInt64(this.view, this.pos);
751-
this.pos += 8;
752-
return value;
753-
}
754-
755-
private readU64AsBigInt(): bigint {
756-
const value = this.view.getBigUint64(this.pos);
746+
private readU64(): number | bigint {
747+
const value = getUint64(this.view, this.pos, this.intMode);
757748
this.pos += 8;
758749
return value;
759750
}
760751

761-
private readI64AsBigInt(): bigint {
762-
const value = this.view.getBigInt64(this.pos);
752+
private readI64(): number | bigint {
753+
const value = getInt64(this.view, this.pos, this.intMode);
763754
this.pos += 8;
764755
return value;
765756
}

‎src/Encoder.ts

+50-9
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { utf8Count, utf8Encode } from "./utils/utf8";
22
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
3-
import { setInt64, setUint64 } from "./utils/int";
43
import { ensureUint8Array } from "./utils/typedArrays";
54
import type { ExtData } from "./ExtData";
65
import type { ContextOf } from "./context";
6+
import { setInt64, setUint64 } from "./utils/int";
77

88
export const DEFAULT_MAX_DEPTH = 100;
99
export const DEFAULT_INITIAL_BUFFER_SIZE = 2048;
@@ -13,13 +13,23 @@ export type EncoderOptions<ContextType = undefined> = Partial<
1313
extensionCodec: ExtensionCodecType<ContextType>;
1414

1515
/**
16-
* Encodes bigint as Int64 or Uint64 if it's set to true.
16+
* Encodes a `number` greater than 32-bit as Int64 or Uint64 if it's set to true, otherwise encode as float64.
17+
*
18+
* Defaults to false.
19+
*/
20+
useInt64: boolean;
21+
22+
/**
23+
* Encodes bigint as Int64 or Uint64 if it's set to true, regardless of the size of bigint number.
1724
* {@link forceIntegerToFloat} does not affect bigint.
1825
* Depends on ES2020's {@link DataView#setBigInt64} and
1926
* {@link DataView#setBigUint64}.
2027
*
2128
* Defaults to false.
2229
*/
30+
forceBigIntToInt64: boolean;
31+
32+
/** @deprecated Alias of `forceBigIntToInt64` */
2333
useBigInt64: boolean;
2434

2535
/**
@@ -43,6 +53,7 @@ export type EncoderOptions<ContextType = undefined> = Partial<
4353
* Defaults to `false`. If enabled, it spends more time in encoding objects.
4454
*/
4555
sortKeys: boolean;
56+
4657
/**
4758
* If `true`, non-integer numbers are encoded in float32, not in float64 (the default).
4859
*
@@ -74,7 +85,8 @@ export type EncoderOptions<ContextType = undefined> = Partial<
7485
export class Encoder<ContextType = undefined> {
7586
private readonly extensionCodec: ExtensionCodecType<ContextType>;
7687
private readonly context: ContextType;
77-
private readonly useBigInt64: boolean;
88+
private readonly forceBigIntToInt64: boolean;
89+
private readonly useInt64: boolean;
7890
private readonly maxDepth: number;
7991
private readonly initialBufferSize: number;
8092
private readonly sortKeys: boolean;
@@ -90,7 +102,8 @@ export class Encoder<ContextType = undefined> {
90102
this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType<ContextType>);
91103
this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined
92104

93-
this.useBigInt64 = options?.useBigInt64 ?? false;
105+
this.forceBigIntToInt64 = options?.forceBigIntToInt64 ?? options?.useBigInt64 ?? false;
106+
this.useInt64 = options?.useInt64 ?? false;
94107
this.maxDepth = options?.maxDepth ?? DEFAULT_MAX_DEPTH;
95108
this.initialBufferSize = options?.initialBufferSize ?? DEFAULT_INITIAL_BUFFER_SIZE;
96109
this.sortKeys = options?.sortKeys ?? false;
@@ -144,8 +157,8 @@ export class Encoder<ContextType = undefined> {
144157
}
145158
} else if (typeof object === "string") {
146159
this.encodeString(object);
147-
} else if (this.useBigInt64 && typeof object === "bigint") {
148-
this.encodeBigInt64(object);
160+
} else if (this.forceBigIntToInt64 && typeof object === "bigint") {
161+
this.encodeBigIntAsInt64(object);
149162
} else {
150163
this.encodeObject(object, depth);
151164
}
@@ -200,7 +213,7 @@ export class Encoder<ContextType = undefined> {
200213
// uint 32
201214
this.writeU8(0xce);
202215
this.writeU32(object);
203-
} else if (!this.useBigInt64) {
216+
} else if (this.useInt64) {
204217
// uint 64
205218
this.writeU8(0xcf);
206219
this.writeU64(object);
@@ -223,7 +236,7 @@ export class Encoder<ContextType = undefined> {
223236
// int 32
224237
this.writeU8(0xd2);
225238
this.writeI32(object);
226-
} else if (!this.useBigInt64) {
239+
} else if (this.useInt64) {
227240
// int 64
228241
this.writeU8(0xd3);
229242
this.writeI64(object);
@@ -248,7 +261,31 @@ export class Encoder<ContextType = undefined> {
248261
}
249262
}
250263

251-
private encodeBigInt64(object: bigint): void {
264+
private encodeBigInt(object: bigint) {
265+
if (object >= 0) {
266+
if (object < 0x100000000 || this.forceIntegerToFloat) {
267+
// uint 32 or lower, or force to float
268+
this.encodeNumber(Number(object));
269+
} else if (object < BigInt("0x10000000000000000")) {
270+
// uint 64
271+
this.encodeBigIntAsInt64(object);
272+
} else {
273+
throw new Error(`Bigint is too large for uint64: ${object}`);
274+
}
275+
} else {
276+
if (object >= -0x80000000 || this.forceIntegerToFloat) {
277+
// int 32 or lower, or force to float
278+
this.encodeNumber(Number(object));
279+
} else if (object >= BigInt(-1) * BigInt("0x8000000000000000")) {
280+
// int 64
281+
this.encodeBigIntAsInt64(object);
282+
} else {
283+
throw new Error(`Bigint is too small for int64: ${object}`);
284+
}
285+
}
286+
}
287+
288+
private encodeBigIntAsInt64(object: bigint): void {
252289
if (object >= BigInt(0)) {
253290
// uint 64
254291
this.writeU8(0xcf);
@@ -300,6 +337,10 @@ export class Encoder<ContextType = undefined> {
300337
this.encodeArray(object, depth);
301338
} else if (ArrayBuffer.isView(object)) {
302339
this.encodeBinary(object);
340+
} else if (typeof object === "bigint") {
341+
// this is here instead of in doEncode so that we can try encoding with an extension first,
342+
// otherwise we would break existing extensions for bigints
343+
this.encodeBigInt(object);
303344
} else if (typeof object === "object") {
304345
this.encodeMap(object as Record<string, unknown>, depth);
305346
} else {

‎src/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import { decode, decodeMulti } from "./decode";
99
export { decode, decodeMulti };
1010
import type { DecodeOptions } from "./decode";
1111
export type { DecodeOptions };
12+
import { IntMode } from './utils/int';
13+
export { IntMode };
1214

1315
import { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream } from "./decodeAsync";
1416
export { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream };

‎src/timestamp.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type
22
import { DecodeError } from "./DecodeError";
3-
import { getInt64, setInt64 } from "./utils/int";
3+
import { IntMode, getInt64, setInt64 } from "./utils/int";
44

55
export const EXT_TIMESTAMP = -1;
66

@@ -87,7 +87,7 @@ export function decodeTimestampToTimeSpec(data: Uint8Array): TimeSpec {
8787
case 12: {
8888
// timestamp 96 = { nsec32 (unsigned), sec64 (signed) }
8989

90-
const sec = getInt64(view, 4);
90+
const sec = getInt64(view, 4, IntMode.UNSAFE_NUMBER);
9191
const nsec = view.getUint32(0);
9292
return { sec, nsec };
9393
}

0 commit comments

Comments
 (0)