From aae44e90b532bc07c147fc03d097cb081653f8df Mon Sep 17 00:00:00 2001 From: GitTomahawk <55877391+GitTomahawk@users.noreply.github.com> Date: Fri, 1 Nov 2024 05:24:02 +0100 Subject: [PATCH 1/5] Update main.js use "insert ignore" to prevent duplicate key issues from dropping th whole query --- main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.js b/main.js index 7797ee1..8a38aed 100644 --- a/main.js +++ b/main.js @@ -1872,7 +1872,7 @@ function pushValuesIntoDB(id, list, cb) { adapter.log.error(error); cb && cb(error); } else { - tasks.push({operation: 'insert', index: sqlDPs[id].index, list, id, callback: cb}); + tasks.push({operation: 'insert ignore', index: sqlDPs[id].index, list, id, callback: cb}); tasks.length === 1 && processTasks(); } } else { From f72ac74b839d2df647068d3f404cab74be0aea35 Mon Sep 17 00:00:00 2001 From: GitTomahawk <55877391+GitTomahawk@users.noreply.github.com> Date: Fri, 1 Nov 2024 05:36:47 +0100 Subject: [PATCH 2/5] Revert "Update main.js" This reverts commit aae44e90b532bc07c147fc03d097cb081653f8df. --- main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.js b/main.js index 8a38aed..7797ee1 100644 --- a/main.js +++ b/main.js @@ -1872,7 +1872,7 @@ function pushValuesIntoDB(id, list, cb) { adapter.log.error(error); cb && cb(error); } else { - tasks.push({operation: 'insert ignore', index: sqlDPs[id].index, list, id, callback: cb}); + tasks.push({operation: 'insert', index: sqlDPs[id].index, list, id, callback: cb}); tasks.length === 1 && processTasks(); } } else { From 7d5dfa5149ae0171cea38e379f11509c00699e97 Mon Sep 17 00:00:00 2001 From: GitTomahawk <55877391+GitTomahawk@users.noreply.github.com> Date: Fri, 1 Nov 2024 05:41:01 +0100 Subject: [PATCH 3/5] Update mysql.js use INSERT IGNORE to prevent duplicate key issues --- lib/mysql.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/mysql.js b/lib/mysql.js index 3ea0c82..ef2d836 100644 --- a/lib/mysql.js +++ b/lib/mysql.js @@ -56,11 +56,11 @@ exports.insert = function (dbname, index, values) { for (let db in insertValues) { if (db === 'ts_counter') { while (insertValues[db].length) { - query += `INSERT INTO \`${dbname}\`.ts_counter (id, ts, val) VALUES ${insertValues[db].splice(0, 500).join(',')};`; + query += `INSERT IGNORE INTO \`${dbname}\`.ts_counter (id, ts, val) VALUES ${insertValues[db].splice(0, 500).join(',')};`; } } else { while (insertValues[db].length) { - query += `INSERT INTO \`${dbname}\`.${db} (id, ts, val, ack, _from, q) VALUES ${insertValues[db].splice(0, 500).join(',')};`; + query += `INSERT IGNORE INTO \`${dbname}\`.${db} (id, ts, val, ack, _from, q) VALUES ${insertValues[db].splice(0, 500).join(',')};`; } } } @@ -88,7 +88,7 @@ exports.getIdSelect = function (dbname, name) { }; exports.getIdInsert = function (dbname, name, type) { - return `INSERT INTO \`${dbname}\`.datapoints (name, type) VALUES('${name}', ${type});`; + return `INSERT IGNORE INTO \`${dbname}\`.datapoints (name, type) VALUES('${name}', ${type});`; }; exports.getIdUpdate = function (dbname, id, type) { @@ -104,7 +104,7 @@ exports.getFromSelect = function (dbname, from) { }; exports.getFromInsert = function (dbname, from) { - return `INSERT INTO \`${dbname}\`.sources (name) VALUES('${from}');`; + return `INSERT IGNORE INTO \`${dbname}\`.sources (name) VALUES('${from}');`; }; exports.getCounterDiff = function (dbname, options) { From b08947394ecc6fabec538e92b1fc223a2f435975 Mon Sep 17 00:00:00 2001 From: GitTomahawk <55877391+GitTomahawk@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:41:30 +0100 Subject: [PATCH 4/5] Update mysql.js aded newline for multi-inserts added IGNORE to Update --- lib/mysql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mysql.js b/lib/mysql.js index ef2d836..18e7739 100644 --- a/lib/mysql.js +++ b/lib/mysql.js @@ -56,11 +56,11 @@ exports.insert = function (dbname, index, values) { for (let db in insertValues) { if (db === 'ts_counter') { while (insertValues[db].length) { - query += `INSERT IGNORE INTO \`${dbname}\`.ts_counter (id, ts, val) VALUES ${insertValues[db].splice(0, 500).join(',')};`; + query += `INSERT IGNORE INTO \`${dbname}\`.ts_counter (id, ts, val) VALUES ${insertValues[db].splice(0, 500).join(',')};\n`; } } else { while (insertValues[db].length) { - query += `INSERT IGNORE INTO \`${dbname}\`.${db} (id, ts, val, ack, _from, q) VALUES ${insertValues[db].splice(0, 500).join(',')};`; + query += `INSERT IGNORE INTO \`${dbname}\`.${db} (id, ts, val, ack, _from, q) VALUES ${insertValues[db].splice(0, 500).join(',')};\n`; } } } @@ -92,7 +92,7 @@ exports.getIdInsert = function (dbname, name, type) { }; exports.getIdUpdate = function (dbname, id, type) { - return `UPDATE \`${dbname}\`.datapoints SET type=${type} WHERE id=${id};`; + return `UPDATE IGNORE \`${dbname}\`.datapoints SET type=${type} WHERE id=${id};`; }; exports.getFromSelect = function (dbname, from) { From ae3db2f00cf61056cfc0582d994d9c8bed6885a0 Mon Sep 17 00:00:00 2001 From: GitTomahawk <55877391+GitTomahawk@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:05:21 +0100 Subject: [PATCH 5/5] Update mysql.js Increased maximum values per insert to prevent query split into multiple inserts (which caused errors) --- lib/mysql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mysql.js b/lib/mysql.js index 18e7739..1f0a466 100644 --- a/lib/mysql.js +++ b/lib/mysql.js @@ -56,11 +56,11 @@ exports.insert = function (dbname, index, values) { for (let db in insertValues) { if (db === 'ts_counter') { while (insertValues[db].length) { - query += `INSERT IGNORE INTO \`${dbname}\`.ts_counter (id, ts, val) VALUES ${insertValues[db].splice(0, 500).join(',')};\n`; + query += `INSERT IGNORE INTO \`${dbname}\`.ts_counter (id, ts, val) VALUES ${insertValues[db].splice(0, 1000).join(',')};\n`; } } else { while (insertValues[db].length) { - query += `INSERT IGNORE INTO \`${dbname}\`.${db} (id, ts, val, ack, _from, q) VALUES ${insertValues[db].splice(0, 500).join(',')};\n`; + query += `INSERT IGNORE INTO \`${dbname}\`.${db} (id, ts, val, ack, _from, q) VALUES ${insertValues[db].splice(0, 1000).join(',')};\n`; } } }