borogove

Keep sortId shorter with a large MessageSync rss

created pr with borogove/sort-id-length.1 on 2026-11-10T03:06:30Z · by 9e834e0d

added borogove/sort-id-length.2 on 2026-11-10T19:38:47Z · by 9e834e0d


                      -:  ------- > 1:  25705f6 Never start a sync window at a franctional sortId


                      1:  0bc2ca2 = 2:  341262a Keep sortId shorter with a large MessageSync

added borogove/sort-id-length.3 on 2026-11-10T20:28:29Z · by 9e834e0d


                      1:  25705f6 ! 1:  07afd44 Never start a sync window at a franctional sortId


                      2:  341262a ! 2:  83a6338 Keep sortId shorter with a large MessageSync

added borogove/sort-id-length.4 on 2026-11-10T20:29:20Z · by 9e834e0d


                      1:  07afd44 ! 1:  309d938 Never start a sync window at a franctional sortId


                      2:  83a6338 = 2:  65784dd Keep sortId shorter with a large MessageSync

cmds

checkout latest patchset:

ssh pr.pico.sh pull borogove:sort-id-length | git am -3

checkout specific patchset revision:

ssh pr.pico.sh pull borogove:sort-id-length [rev] | git am -3

add changes to patch request:

git format-patch main --stdout | ssh pr.pico.sh borogove:sort-id-length

add comment to patch request:

ssh pr.pico.sh comment borogove:sort-id-length "lgtm!"

Patchset `borogove/sort-id-length.4` on 2026-11-10T20:29:20Z · commit 309d938

→ Never start a sync window at a franctional sortId

Stephen Paul Weber 2026-11-10T02:48:00Z

Keep sortId shorter with a large MessageSync

Stephen Paul Weber 2026-11-10T03:03:36Z

Never start a sync window at a franctional sortId

Stephen Paul Weber 2026-11-10T02:48:00Z

Instead, advance to the next short/"integer" sortId as the bottom of the
window to keep ids shorter.

Semantic diff summary

0 added, 0 modified, 0 signature changed, 0 removed across 0 analyzed files (2 files skipped: unsupported file type)

+2 -4 borogove/Chat.hx #

......@@ -2024,10 +2024,8 @@ class Channel extends Chat {
		}

		// Sort into a window so live messages arriving concurrently can come after
		final sortFrom = sortA ?? syncPoint?.sortId;
		final sortNext = FractionalIndexing.between(sortFrom, null, FractionalIndexing.BASE_95_DIGITS);
		final sortIdNext = FractionalIndexing.between(sortId ?? sortNext, null, FractionalIndexing.BASE_95_DIGITS);
		final sortTo = sortNext < sortIdNext ? sortNext : sortIdNext;
		final sortFrom = FractionalIndexing.between(sortA ?? syncPoint?.sortId, null, FractionalIndexing.BASE_95_DIGITS);
		final sortTo = FractionalIndexing.between(sortFrom, null, FractionalIndexing.BASE_95_DIGITS);
		if (sortId == null || sortId <= sortTo) sortId = FractionalIndexing.between(sortTo, null, FractionalIndexing.BASE_95_DIGITS);

		var threeDaysAgo = Date.format(

+104 -0 test/TestSortId.hx #

......@@ -361,4 +361,108 @@ class TestSortId extends utest.Test {
			);
		});
	}

	public function testChannelSyncBatchesKeepSortIdsShort(async: Async) {
		final persistence = new Dummy();
		final client = new Client("test@example.com", persistence);
		final disco = new Caps("", [], ["http://jabber.org/protocol/muc", "urn:xmpp:mam:2"], []);
		final channel = new Channel(client, client.stream, persistence, "batchsync@example.com", Open, false, false, null, null, null, disco);
		client.chats.push(channel);

		final batchCount = 12;
		final messagesPerBatch = 20;
		final maxAllowedLength = 12;

		var currentBatch = 0;
		var completedBatches = 0;
		var maxSortIdLength = 0;
		var previousSortId: Null<String> = null;
		var lastSyncedMessage: Null<ChatMessage> = null;
		var syncPoint: Null<ChatMessage> = null;
		var pendingIqId: Null<String> = null;
		var pendingQueryId: Null<String> = null;

		var startBatch:() -> Void = null;
		var feedBatch:() -> Void = null;

		client.addSyncMessageListener((message) -> {
			final sortId = message.sortId;
			if (sortId == null) return;

			if (previousSortId != null) {
				Assert.isTrue(previousSortId < sortId, 'sync sortIds should keep increasing ($previousSortId < $sortId)');
			}
			previousSortId = sortId;
			if (sortId.length > maxSortIdLength) maxSortIdLength = sortId.length;
			lastSyncedMessage = message;
		});

		client.on("chats/update", (chats) -> {
			if (channel.sync != null) return EventHandled;

			completedBatches++;
			if (completedBatches >= batchCount) {
				Assert.isTrue(maxSortIdLength <= maxAllowedLength, 'max sortId length $maxSortIdLength should be <= $maxAllowedLength after $batchCount sync batches');
				async.done();
				return EventHandled;
			}

			Assert.notNull(lastSyncedMessage, "Batch should have synced at least one message");
			syncPoint = lastSyncedMessage;
			lastSyncedMessage = null;
			startBatch();
			return EventHandled;
		});

		client.stream.on("sendStanza", (stanza: Stanza) -> {
			if (stanza.name != "iq") return EventUnhandled;
			final query = stanza.findChild("{urn:xmpp:mam:2}query");
			if (query == null) return EventUnhandled;

			pendingIqId = stanza.attr.get("id");
			pendingQueryId = query.attr.get("queryid");
			haxe.Timer.delay(() -> feedBatch(), 0);
			return EventHandled;
		});

		startBatch = function() {
			if (currentBatch >= batchCount) return;
			currentBatch++;
			channel.doSync(syncPoint);
		};

		feedBatch = function() {
			final iqId = pendingIqId;
			final queryId = pendingQueryId;
			if (iqId == null || queryId == null) {
				Assert.fail("MAM query was not sent");
				return;
			}

			for (i in 0...messagesPerBatch) {
				final serverId = 'batch${currentBatch}-msg$i';
				client.stream.onStanza(new Stanza("message", { from: channel.chatId, xmlns: "jabber:client" })
					.tag("result", { xmlns: "urn:xmpp:mam:2", queryid: queryId, id: serverId })
						.tag("forwarded", { xmlns: "urn:xmpp:forward:0" })
							.tag("delay", { xmlns: "urn:xmpp:delay", stamp: "2023-01-01T00:00:00Z" }).up()
							.tag("message", { from: channel.chatId + "/user", to: "test@example.com", id: serverId, xmlns: "jabber:client" })
								.textTag("body", 'message $serverId')
							.up()
						.up()
					.up()
				);
			}

			client.stream.onStanza(new Stanza("iq", { type: "result", id: iqId, from: channel.chatId, xmlns: "jabber:client" })
				.tag("fin", { xmlns: "urn:xmpp:mam:2", complete: "true" })
					.tag("set", { xmlns: "http://jabber.org/protocol/rsm" })
						.textTag("first", 'batch${currentBatch}-msg0')
						.textTag("last", 'batch${currentBatch}-msg${messagesPerBatch - 1}')
					.up()
				.up()
			);
		};

		startBatch();
	}
}

...	...		@@ -361,4 +361,108 @@ class TestSortId extends utest.Test {
361	361		);
362	362		});
363	363		}
	364	+
	365	+	public function testChannelSyncBatchesKeepSortIdsShort(async: Async) {
	366	+	final persistence = new Dummy();
	367	+	final client = new Client("test@example.com", persistence);
	368	+	final disco = new Caps("", [], ["http://jabber.org/protocol/muc", "urn:xmpp:mam:2"], []);
	369	+	final channel = new Channel(client, client.stream, persistence, "batchsync@example.com", Open, false, false, null, null, null, disco);
	370	+	client.chats.push(channel);
	371	+
	372	+	final batchCount = 12;
	373	+	final messagesPerBatch = 20;
	374	+	final maxAllowedLength = 12;
	375	+
	376	+	var currentBatch = 0;
	377	+	var completedBatches = 0;
	378	+	var maxSortIdLength = 0;
	379	+	var previousSortId: Null<String> = null;
	380	+	var lastSyncedMessage: Null<ChatMessage> = null;
	381	+	var syncPoint: Null<ChatMessage> = null;
	382	+	var pendingIqId: Null<String> = null;
	383	+	var pendingQueryId: Null<String> = null;
	384	+
	385	+	var startBatch:() -> Void = null;
	386	+	var feedBatch:() -> Void = null;
	387	+
	388	+	client.addSyncMessageListener((message) -> {
	389	+	final sortId = message.sortId;
	390	+	if (sortId == null) return;
	391	+
	392	+	if (previousSortId != null) {
	393	+	Assert.isTrue(previousSortId < sortId, 'sync sortIds should keep increasing ($previousSortId < $sortId)');
	394	+	}
	395	+	previousSortId = sortId;
	396	+	if (sortId.length > maxSortIdLength) maxSortIdLength = sortId.length;
	397	+	lastSyncedMessage = message;
	398	+	});
	399	+
	400	+	client.on("chats/update", (chats) -> {
	401	+	if (channel.sync != null) return EventHandled;
	402	+
	403	+	completedBatches++;
	404	+	if (completedBatches >= batchCount) {
	405	+	Assert.isTrue(maxSortIdLength <= maxAllowedLength, 'max sortId length $maxSortIdLength should be <= $maxAllowedLength after $batchCount sync batches');
	406	+	async.done();
	407	+	return EventHandled;
	408	+	}
	409	+
	410	+	Assert.notNull(lastSyncedMessage, "Batch should have synced at least one message");
	411	+	syncPoint = lastSyncedMessage;
	412	+	lastSyncedMessage = null;
	413	+	startBatch();
	414	+	return EventHandled;
	415	+	});
	416	+
	417	+	client.stream.on("sendStanza", (stanza: Stanza) -> {
	418	+	if (stanza.name != "iq") return EventUnhandled;
	419	+	final query = stanza.findChild("{urn:xmpp:mam:2}query");
	420	+	if (query == null) return EventUnhandled;
	421	+
	422	+	pendingIqId = stanza.attr.get("id");
	423	+	pendingQueryId = query.attr.get("queryid");
	424	+	haxe.Timer.delay(() -> feedBatch(), 0);
	425	+	return EventHandled;
	426	+	});
	427	+
	428	+	startBatch = function() {
	429	+	if (currentBatch >= batchCount) return;
	430	+	currentBatch++;
	431	+	channel.doSync(syncPoint);
	432	+	};
	433	+
	434	+	feedBatch = function() {
	435	+	final iqId = pendingIqId;
	436	+	final queryId = pendingQueryId;
	437	+	if (iqId == null \|\| queryId == null) {
	438	+	Assert.fail("MAM query was not sent");
	439	+	return;
	440	+	}
	441	+
	442	+	for (i in 0...messagesPerBatch) {
	443	+	final serverId = 'batch${currentBatch}-msg$i';
	444	+	client.stream.onStanza(new Stanza("message", { from: channel.chatId, xmlns: "jabber:client" })
	445	+	.tag("result", { xmlns: "urn:xmpp:mam:2", queryid: queryId, id: serverId })
	446	+	.tag("forwarded", { xmlns: "urn:xmpp:forward:0" })
	447	+	.tag("delay", { xmlns: "urn:xmpp:delay", stamp: "2023-01-01T00:00:00Z" }).up()
	448	+	.tag("message", { from: channel.chatId + "/user", to: "test@example.com", id: serverId, xmlns: "jabber:client" })
	449	+	.textTag("body", 'message $serverId')
	450	+	.up()
	451	+	.up()
	452	+	.up()
	453	+	);
	454	+	}
	455	+
	456	+	client.stream.onStanza(new Stanza("iq", { type: "result", id: iqId, from: channel.chatId, xmlns: "jabber:client" })
	457	+	.tag("fin", { xmlns: "urn:xmpp:mam:2", complete: "true" })
	458	+	.tag("set", { xmlns: "http://jabber.org/protocol/rsm" })
	459	+	.textTag("first", 'batch${currentBatch}-msg0')
	460	+	.textTag("last", 'batch${currentBatch}-msg${messagesPerBatch - 1}')
	461	+	.up()
	462	+	.up()
	463	+	);
	464	+	};
	465	+
	466	+	startBatch();
	467	+	}
364	468		}

...	...		@@ -2024,10 +2024,8 @@ class Channel extends Chat {
2024	2024		}
2025	2025
2026	2026		// Sort into a window so live messages arriving concurrently can come after
2027		-	final sortFrom = sortA ?? syncPoint?.sortId;
2028		-	final sortNext = FractionalIndexing.between(sortFrom, null, FractionalIndexing.BASE_95_DIGITS);
2029		-	final sortIdNext = FractionalIndexing.between(sortId ?? sortNext, null, FractionalIndexing.BASE_95_DIGITS);
2030		-	final sortTo = sortNext < sortIdNext ? sortNext : sortIdNext;
	2027	+	final sortFrom = FractionalIndexing.between(sortA ?? syncPoint?.sortId, null, FractionalIndexing.BASE_95_DIGITS);
	2028	+	final sortTo = FractionalIndexing.between(sortFrom, null, FractionalIndexing.BASE_95_DIGITS);
2031	2029		if (sortId == null \|\| sortId <= sortTo) sortId = FractionalIndexing.between(sortTo, null, FractionalIndexing.BASE_95_DIGITS);
2032	2030
2033	2031		var threeDaysAgo = Date.format(

borogove

Patchset borogove/sort-id-length.4 on 2026-11-10T20:29:20Z · commit 309d938

Patchset `borogove/sort-id-length.4` on 2026-11-10T20:29:20Z · commit 309d938