batch auctiondb channel sync

TradeSkillMaster_AuctionDB/Modules/ChannelSync.lua

@@ -17,6 +17,7 @@ local CHUNK_SIZE = 220
 local BUNDLE_TIMEOUT = 45
 local SEND_INTERVAL = 0.08
 local MAX_TOTAL_CHUNKS = 800
+local MAX_ITEMS_PER_PAYLOAD = 250
 
 local private = {
 	channelId = nil,
@@ -117,42 +118,68 @@ function ChannelSync:BroadcastScanData(scanType, items)
 	EnsureChannel()
 	if not private.channelName or not private.channelId then return end
 
-	local payload = ChannelSync:BuildPayload(scanType, items)
+	local itemIDs = ChannelSync:CollectItemIDs(items)
-	if not payload then return end
+	if not itemIDs or #itemIDs == 0 then return end
-	local encoded = EncodePayload(payload)
+
-	if not encoded then return end
+	local function QueueEncoded(encoded)
-
+		local hash = HashString(encoded)
-	local hash = HashString(encoded)
+		if hash == private.lastBroadcastHash then return end
-	if hash == private.lastBroadcastHash then return end
+		private.lastBroadcastHash = hash
-	private.lastBroadcastHash = hash
+		local total = ceil(#encoded / CHUNK_SIZE)
-	local total = ceil(#encoded / CHUNK_SIZE)
+		if total > MAX_TOTAL_CHUNKS then return false end
-	if total > MAX_TOTAL_CHUNKS then
+		tinsert(private.sendQueue, {hash = hash, encoded = encoded, total = total})
-		TSM:Print("AuctionDB channel sync payload too large; skipping broadcast.")
+		if not private.isSending then
-		return
+			private.isSending = true
+			TSMAPI.Threading:Start(ChannelSync.SendQueueThread, 0.3)
+		end
+		return true
+	end
+
+	local function QueueBatch(startIndex, endIndex)
+		local batch = {}
+		for i = startIndex, endIndex do
+			tinsert(batch, itemIDs[i])
+		end
+		local payload = ChannelSync:BuildPayloadFromItemIDs(scanType, batch)
+		if not payload then return end
+		local encoded = EncodePayload(payload)
+		if not encoded then return end
+		local total = ceil(#encoded / CHUNK_SIZE)
+		if total > MAX_TOTAL_CHUNKS and #batch > 1 then
+			local mid = floor((startIndex + endIndex) / 2)
+			QueueBatch(startIndex, mid)
+			QueueBatch(mid + 1, endIndex)
+			return
+		end
+		if total > MAX_TOTAL_CHUNKS then
+			TSM:Print("AuctionDB channel sync payload too large; skipping batch.")
+			return
+		end
+		QueueEncoded(encoded)
 	end
 
-	tinsert(private.sendQueue, {hash = hash, encoded = encoded, total = total})
+	for i = 1, #itemIDs, MAX_ITEMS_PER_PAYLOAD do
-	if not private.isSending then
+		QueueBatch(i, min(i + MAX_ITEMS_PER_PAYLOAD - 1, #itemIDs))
-		private.isSending = true
-		TSMAPI.Threading:Start(ChannelSync.SendQueueThread, 0.3)
 	end
 end
 
-function ChannelSync:BuildPayload(scanType, items)
+function ChannelSync:CollectItemIDs(items)
-	local itemIDs = {}
+	local itemIDs, list = {}, {}
 	if items then
 		if #items > 0 then
 			for _, itemString in ipairs(items) do
 				local itemID = TSMAPI:GetItemID(itemString)
-				if itemID then
+				if itemID and not itemIDs[itemID] then
 					itemIDs[itemID] = true
+					tinsert(list, itemID)
 				end
 			end
 		else
 			for itemString in pairs(items) do
 				local itemID = TSMAPI:GetItemID(itemString)
-				if itemID then
+				if itemID and not itemIDs[itemID] then
 					itemIDs[itemID] = true
+					tinsert(list, itemID)
 				end
 			end
 		end
@@ -161,13 +188,16 @@ function ChannelSync:BuildPayload(scanType, items)
 		for itemID in pairs(TSM.data) do
 			TSM:DecodeItemData(itemID)
 			if TSM.data[itemID].lastScan == scanTime then
-				itemIDs[itemID] = true
+				tinsert(list, itemID)
 			end
 		end
 	end
+	return list
+end
 
+function ChannelSync:BuildPayloadFromItemIDs(scanType, itemIDList)
 	local payloadItems = {}
-	for itemID in pairs(itemIDs) do
+	for _, itemID in ipairs(itemIDList) do
 		local data = TSM.data[itemID]
 		if data then
 			TSM:EncodeItemData(itemID)