From d07bde9e907d6b8a7484295d94f6a152400fd546 Mon Sep 17 00:00:00 2001 From: Jason Weathered Date: Tue, 8 Dec 2020 17:12:43 +1000 Subject: [PATCH 1/4] Test with PostgreSQL 12 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75f6ded8..bcd854dd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ jobs: matrix: ruby_version: [2.5.x, 2.6.x, 2.7.x] gemfile: ["4.2", "5.2", "6.0"] - postgres_version: [9, 10, 11] + postgres_version: [9, 10, 11, 12] exclude: - { gemfile: "4.2", ruby_version: "2.7.x" } services: From 9de6176928c603e4c840513faf35832f5c8fb613 Mon Sep 17 00:00:00 2001 From: Jason Weathered Date: Tue, 8 Dec 2020 16:33:25 +1000 Subject: [PATCH 2/4] Replace use of obsolete `adsrc` with `pg_get_expr(adbin, adrelid)` PostgreSQL 12 drops this obsolete column. https://www.postgresql.org/docs/12/release-12.html: > Remove obsolete pg_attrdef.adsrc column (Peter Eisentraut) > > This column has been deprecated for a long time, because it > did not update in response to other catalog changes (such as > column renamings). The recommended way to get a text version > of a default-value expression from `pg_attrdef` is > `pg_get_expr(adbin, adrelid)`. --- lib/que/migrations.spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/que/migrations.spec.rb b/lib/que/migrations.spec.rb index 83120098..15eeb026 100644 --- a/lib/que/migrations.spec.rb +++ b/lib/que/migrations.spec.rb @@ -9,7 +9,7 @@ default = proc do result = Que.execute <<-SQL - select adsrc::integer + select pg_get_expr(adbin, adrelid)::integer AS adsrc from pg_attribute a join pg_class c on c.oid = a.attrelid join pg_attrdef on adrelid = attrelid AND adnum = attnum From 386dce553206b3f858923f1aa724b5f6038e4fd4 Mon Sep 17 00:00:00 2001 From: Jason Weathered Date: Tue, 8 Dec 2020 17:11:42 +1000 Subject: [PATCH 3/4] Add support for retrieving PostgreSQL server version --- lib/que/connection.rb | 4 ++++ lib/que/connection.spec.rb | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/lib/que/connection.rb b/lib/que/connection.rb index 30adf777..820562da 100644 --- a/lib/que/connection.rb +++ b/lib/que/connection.rb @@ -119,6 +119,10 @@ def drain_notifications loop { break if next_notification.nil? } end + def server_version + wrapped_connection.server_version + end + def in_transaction? wrapped_connection.transaction_status != ::PG::PQTRANS_IDLE end diff --git a/lib/que/connection.spec.rb b/lib/que/connection.spec.rb index 74636d62..0379cf01 100644 --- a/lib/que/connection.spec.rb +++ b/lib/que/connection.spec.rb @@ -45,6 +45,14 @@ end end + describe "server_version" do + it "returns the PostgreSQL server version" do + assert_instance_of Integer, connection.server_version + assert_operator connection.server_version, :>=, 9_06_00 + assert_operator connection.server_version, :<, 99_00_00 + end + end + describe "in_transaction?" do it "should know when it is in a transaction" do refute connection.in_transaction? From 2c2687ffd47c1209b01863f07626840987452318 Mon Sep 17 00:00:00 2001 From: Jason Weathered Date: Wed, 9 Dec 2020 12:41:02 +1000 Subject: [PATCH 4/4] Fix lock leak on PostgreSQL 12 and later PostgreSQL 12 and later do not materialise all CTEs automatically. This can result in multiple scans occurring which in turn can result in `pg_try_advisory_lock` being invoked multiple times on the same job. This results in a leak as we only call unlock once. --- lib/que/locker.rb | 4 +++- lib/que/locker.spec.rb | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/que/locker.rb b/lib/que/locker.rb index 202ae899..651e6ec5 100644 --- a/lib/que/locker.rb +++ b/lib/que/locker.rb @@ -393,10 +393,12 @@ def lock_jobs(metajobs) } end + materalize_cte = connection.server_version >= 12_00_00 + jobs = connection.execute \ <<-SQL - WITH jobs AS (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')})) + WITH jobs AS #{materalize_cte ? 'MATERIALIZED' : ''} (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')})) SELECT * FROM jobs WHERE pg_try_advisory_lock(id) SQL diff --git a/lib/que/locker.spec.rb b/lib/que/locker.spec.rb index 91be86cd..50172f4c 100644 --- a/lib/que/locker.spec.rb +++ b/lib/que/locker.spec.rb @@ -239,6 +239,23 @@ def assert_que_locker_insertion( locker.stop! end + it "locks only accepted jobs in a listen batch" do + locker_settings[:poll] = false + locker + sleep_until_equal(1) { DB[:que_lockers].count } + + Que.execute <<~SQL + INSERT INTO que_jobs (job_class, priority) + SELECT 'Que::Job', 1 + FROM generate_series(1, 10) AS i; + SQL + + sleep_until_equal(2) { active_jobs_dataset.count } + sleep_until_equal(0) { locked_ids.size } + + locker.stop! + end + it "should repeat batch polls until there are no more available jobs" do Que.execute <<-SQL INSERT INTO que_jobs (job_class, priority)