From d31c92943c942234b49b4fb4a3f30739893f43dc Mon Sep 17 00:00:00 2001 From: Daria Mayorova Date: Thu, 24 Jul 2025 15:34:43 +0200 Subject: [PATCH 01/11] Change bundler version to 2.3.27 to match the system bundler in the base image --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d4755b8a..6fff3e10 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -399,4 +399,4 @@ DEPENDENCIES zeitwerk (~> 2.6.18) BUNDLED WITH - 2.4.17 + 2.3.27 From 7c123b6d5d81851a1e0f44f744551ce577804d12 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Wed, 8 Oct 2025 17:14:00 +0300 Subject: [PATCH 02/11] avoid MD5 for FIPS support This is a temporary patch until upstream fix is merged, see https://github.com/que-rb/que/pull/437 --- ...ace_md5_with_hashtext_in_que_job_notify.rb | 134 ++++++++++++++++++ db/structure-10.sql | 7 +- db/structure.sql | 7 +- 3 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 db/migrate/20251008141931_replace_md5_with_hashtext_in_que_job_notify.rb diff --git a/db/migrate/20251008141931_replace_md5_with_hashtext_in_que_job_notify.rb b/db/migrate/20251008141931_replace_md5_with_hashtext_in_que_job_notify.rb new file mode 100644 index 00000000..0b512d77 --- /dev/null +++ b/db/migrate/20251008141931_replace_md5_with_hashtext_in_que_job_notify.rb @@ -0,0 +1,134 @@ +class ReplaceMd5WithHashtextInQueJobNotify < ActiveRecord::Migration[7.0] + # This fixes https://github.com/que-rb/que/pull/437 + # Be careful on Que upgrade in case the final fix differed. + + def up + Que.transaction do + Que.execute <<~SQL + CREATE OR REPLACE FUNCTION que_job_notify() RETURNS trigger AS $$ + DECLARE + locker_pid integer; + sort_key json; + BEGIN + -- Don't do anything if the job is scheduled for a future time. + IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN + RETURN null; + END IF; + + -- Pick a locker to notify of the job's insertion, weighted by their number + -- of workers. Should bounce pseudorandomly between lockers on each + -- invocation, hence the hashtext-ordering, but still touch each one equally, + -- hence the modulo using the job_id. + SELECT pid + INTO locker_pid + FROM ( + SELECT *, last_value(row_number) OVER () + 1 AS count + FROM ( + SELECT *, row_number() OVER () - 1 AS row_number + FROM ( + SELECT * + FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id + WHERE + listening AND + queues @> ARRAY[NEW.queue] AND + ql.job_schema_version = NEW.job_schema_version + ORDER BY hashtext(pid::text || id::text) + ) t1 + ) t2 + ) t3 + WHERE NEW.id % count = row_number; + + IF locker_pid IS NOT NULL THEN + -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so + -- rather than throw errors when someone enqueues a big job, just + -- broadcast the most pertinent information, and let the locker query for + -- the record after it's taken the lock. The worker will have to hit the + -- DB in order to make sure the job is still visible anyway. + SELECT row_to_json(t) + INTO sort_key + FROM ( + SELECT + 'job_available' AS message_type, + NEW.queue AS queue, + NEW.priority AS priority, + NEW.id AS id, + -- Make sure we output timestamps as UTC ISO 8601 + to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at + ) t; + + PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text); + END IF; + + RETURN null; + END + $$ + LANGUAGE plpgsql; + SQL + end + end + + def down + Que.transaction do + Que.execute <<~SQL + CREATE OR REPLACE FUNCTION que_job_notify() RETURNS trigger AS $$ + DECLARE + locker_pid integer; + sort_key json; + BEGIN + -- Don't do anything if the job is scheduled for a future time. + IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN + RETURN null; + END IF; + + -- Pick a locker to notify of the job's insertion, weighted by their number + -- of workers. Should bounce pseudorandomly between lockers on each + -- invocation, hence the md5-ordering, but still touch each one equally, + -- hence the modulo using the job_id. + SELECT pid + INTO locker_pid + FROM ( + SELECT *, last_value(row_number) OVER () + 1 AS count + FROM ( + SELECT *, row_number() OVER () - 1 AS row_number + FROM ( + SELECT * + FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id + WHERE + listening AND + queues @> ARRAY[NEW.queue] AND + ql.job_schema_version = NEW.job_schema_version + ORDER BY md5(pid::text || id::text) + ) t1 + ) t2 + ) t3 + WHERE NEW.id % count = row_number; + + IF locker_pid IS NOT NULL THEN + -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so + -- rather than throw errors when someone enqueues a big job, just + -- broadcast the most pertinent information, and let the locker query for + -- the record after it's taken the lock. The worker will have to hit the + -- DB in order to make sure the job is still visible anyway. + SELECT row_to_json(t) + INTO sort_key + FROM ( + SELECT + 'job_available' AS message_type, + NEW.queue AS queue, + NEW.priority AS priority, + NEW.id AS id, + -- Make sure we output timestamps as UTC ISO 8601 + to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at + ) t; + + PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text); + END IF; + + RETURN null; + END + $$ + LANGUAGE plpgsql; + SQL + end + end +end diff --git a/db/structure-10.sql b/db/structure-10.sql index de6716bb..acb44596 100644 --- a/db/structure-10.sql +++ b/db/structure-10.sql @@ -112,7 +112,7 @@ CREATE FUNCTION public.que_job_notify() RETURNS trigger -- Pick a locker to notify of the job's insertion, weighted by their number -- of workers. Should bounce pseudorandomly between lockers on each - -- invocation, hence the md5-ordering, but still touch each one equally, + -- invocation, hence the hashtext-ordering, but still touch each one equally, -- hence the modulo using the job_id. SELECT pid INTO locker_pid @@ -127,7 +127,7 @@ CREATE FUNCTION public.que_job_notify() RETURNS trigger listening AND queues @> ARRAY[NEW.queue] AND ql.job_schema_version = NEW.job_schema_version - ORDER BY md5(pid::text || id::text) + ORDER BY hashtext(pid::text || id::text) ) t1 ) t2 ) t3 @@ -1527,6 +1527,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20230629131935'), ('20230703133544'), ('20230703134109'), -('20230704131552'); +('20230704131552'), +('20251008141931'); diff --git a/db/structure.sql b/db/structure.sql index d24d2102..7cb27aa3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -114,7 +114,7 @@ CREATE FUNCTION public.que_job_notify() RETURNS trigger -- Pick a locker to notify of the job's insertion, weighted by their number -- of workers. Should bounce pseudorandomly between lockers on each - -- invocation, hence the md5-ordering, but still touch each one equally, + -- invocation, hence the hashtext-ordering, but still touch each one equally, -- hence the modulo using the job_id. SELECT pid INTO locker_pid @@ -129,7 +129,7 @@ CREATE FUNCTION public.que_job_notify() RETURNS trigger listening AND queues @> ARRAY[NEW.queue] AND ql.job_schema_version = NEW.job_schema_version - ORDER BY md5(pid::text || id::text) + ORDER BY hashtext(pid::text || id::text) ) t1 ) t2 ) t3 @@ -1529,6 +1529,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20230629131935'), ('20230703133544'), ('20230703134109'), -('20230704131552'); +('20230704131552'), +('20251008141931'); From 05cf22e5398057a17e099de4836b7cca4dd2d38d Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 00:32:41 +0000 Subject: [PATCH 03/11] Update rack to version 2.2.13 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6fff3e10..4c024818 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -241,7 +241,7 @@ GEM que (>= 1) sinatra racc (1.8.1) - rack (2.2.12) + rack (2.2.13) rack-protection (2.2.3) rack rack-test (2.1.0) From adfa0d00e9dc22d1eee2d9dbfcea9d863d7c1479 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 16:32:35 +0000 Subject: [PATCH 04/11] Update rack to version 2.2.14 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4c024818..32883950 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -241,7 +241,7 @@ GEM que (>= 1) sinatra racc (1.8.1) - rack (2.2.13) + rack (2.2.14) rack-protection (2.2.3) rack rack-test (2.1.0) From f3f426c5c527d9bb2a72957a0a19c445e871504d Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Tue, 7 Oct 2025 18:23:48 +0000 Subject: [PATCH 05/11] Update rack to version 2.2.19 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 32883950..2f2d692f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -241,7 +241,7 @@ GEM que (>= 1) sinatra racc (1.8.1) - rack (2.2.14) + rack (2.2.19) rack-protection (2.2.3) rack rack-test (2.1.0) From e1bcca34d918ddae22fcae7e0b35d84591446584 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 18:08:40 +0000 Subject: [PATCH 06/11] Update rack to version 2.2.20 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2f2d692f..66baf9b6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -241,7 +241,7 @@ GEM que (>= 1) sinatra racc (1.8.1) - rack (2.2.19) + rack (2.2.20) rack-protection (2.2.3) rack rack-test (2.1.0) From f63551c8ccbb8516e9aad4424e271ebc98fd0214 Mon Sep 17 00:00:00 2001 From: Daria Mayorova Date: Mon, 13 Apr 2026 15:45:27 +0200 Subject: [PATCH 07/11] Upgrade rack to 2.2.23 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 66baf9b6..7d4dfe65 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -241,7 +241,7 @@ GEM que (>= 1) sinatra racc (1.8.1) - rack (2.2.20) + rack (2.2.23) rack-protection (2.2.3) rack rack-test (2.1.0) From 635192db82c1be870083c1a686f8baea8d8a75bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 11 Dec 2025 09:19:37 +0100 Subject: [PATCH 08/11] Que: add `connection_url` parameter to call It's the only way for it to receive the SSL parameters Co-Authored-By: Claude (cherry picked from commit 317aa3f05b33146929dc31ad69793f82828d63fa) --- lib/que/db_connection_url.rb | 52 ++++++++++++++++++++++++++++++++++++ lib/tasks/que.rake | 10 ++++++- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 lib/que/db_connection_url.rb diff --git a/lib/que/db_connection_url.rb b/lib/que/db_connection_url.rb new file mode 100644 index 00000000..dfc0bab9 --- /dev/null +++ b/lib/que/db_connection_url.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Que + class DBConnectionURL + def self.build_connection_url(db_config) + # Build base URL + # Workaround for https://github.com/que-rb/que/issues/442 + connection_url = "#{db_config[:adapter]}://" + + host = db_config[:host] + is_unix_socket = host && host.start_with?('/') + is_ipv6 = host && !is_unix_socket && host.include?(':') + + # Add username and password + connection_url += "#{db_config[:username]}" if db_config[:username] + connection_url += ":#{db_config[:password]}" if db_config[:password] + connection_url += "@" if db_config[:username] || db_config[:password] + + # For Unix sockets, leave host empty; for TCP, add host and port + if is_unix_socket + connection_url += "/" + else + # Wrap IPv6 addresses in square brackets + formatted_host = is_ipv6 ? "[#{host}]" : (host || 'localhost') + connection_url += formatted_host + connection_url += ":#{db_config[:port]}" if db_config[:port] + connection_url += "/" + end + + connection_url += db_config[:database] + + # Build query parameters + params = [] + + # For Unix sockets, add host and port as query parameters + if is_unix_socket + params << "host=#{host}" + params << "port=#{db_config[:port]}" if db_config[:port] + end + + # Add SSL parameters + params << "sslmode=#{db_config[:sslmode]}" if db_config[:sslmode] + params << "sslrootcert=#{db_config[:sslrootcert]}" if db_config[:sslrootcert] + params << "sslcert=#{db_config[:sslcert]}" if db_config[:sslcert] + params << "sslkey=#{db_config[:sslkey]}" if db_config[:sslkey] + + connection_url += "?#{params.join('&')}" if params.any? + + connection_url + end + end +end diff --git a/lib/tasks/que.rake b/lib/tasks/que.rake index 1972d9ef..f0c05f27 100644 --- a/lib/tasks/que.rake +++ b/lib/tasks/que.rake @@ -1,11 +1,18 @@ # frozen_string_literal: true +require 'que/db_connection_url' + task que: 'que:exec' namespace :que do desc 'Start que worker' task exec: :environment do |_, args| - exec("que ./config/environment.rb que/prometheus #{args.extras.join}") + # Build base URL + # Workaround for https://github.com/que-rb/que/issues/442 + db_config = ActiveRecord::Base.connection_db_config.configuration_hash + connection_url = Que::DBConnectionURL.build_connection_url(db_config) + + exec("que --connection-url '#{connection_url}' ./config/environment.rb que/prometheus #{args.extras.join}") end desc 'Reschedule all jobs to be executed now' @@ -20,3 +27,4 @@ namespace :que do Model.find_each(&UpdateJob.method(:perform_later)) end end + From 760b367362c2d4e2f281de69eb79569c90686764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 16 Dec 2025 14:28:57 +0100 Subject: [PATCH 09/11] Add tests Co-Authored-By: Claude (cherry picked from commit 6527ae9b1210cb7a38a96110b657f89219850657) --- test/lib/que/db_connection_url_test.rb | 318 +++++++++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 test/lib/que/db_connection_url_test.rb diff --git a/test/lib/que/db_connection_url_test.rb b/test/lib/que/db_connection_url_test.rb new file mode 100644 index 00000000..88a9a867 --- /dev/null +++ b/test/lib/que/db_connection_url_test.rb @@ -0,0 +1,318 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'que/db_connection_url' + +class Que::DBConnectionURLTest < ActiveSupport::TestCase + test 'minimal configuration' do + db_config = { + adapter: 'postgresql', + database: 'postgres' + } + + expected = 'postgresql://localhost/postgres' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with port' do + db_config = { + adapter: 'postgresql', + host: 'localhost', + port: 5432, + database: 'mydb' + } + + expected = 'postgresql://localhost:5432/mydb' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with username only' do + db_config = { + adapter: 'postgresql', + username: 'user', + host: 'localhost', + database: 'mydb' + } + + expected = 'postgresql://user@localhost/mydb' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with empty password' do + db_config = { + adapter: 'postgresql', + username: 'user', + password: '', + host: 'localhost', + database: 'mydb' + } + + expected = 'postgresql://user:@localhost/mydb' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with username and password' do + db_config = { + adapter: 'postgresql', + username: 'user', + password: 'secret', + host: 'localhost', + database: 'database' + } + + expected = 'postgresql://user:secret@localhost/database' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with password containing special characters' do + db_config = { + adapter: 'postgresql', + username: 'user', + password: 'p@ssw0rd!#$', + host: 'localhost', + database: 'mydb' + } + + expected = 'postgresql://user:p@ssw0rd!#$@localhost/mydb' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'full connection with all basic parameters' do + db_config = { + adapter: 'postgresql', + username: 'user', + password: 'password', + host: 'localhost', + port: 5432, + database: 'database' + } + + expected = 'postgresql://user:password@localhost:5432/database' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with different host' do + db_config = { + adapter: 'postgresql', + username: 'username', + host: 'hostname', + database: 'databasename' + } + + expected = 'postgresql://username@hostname/databasename' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with SSL root certificate only' do + db_config = { + adapter: 'postgresql', + host: 'localhost', + database: 'mydb', + sslrootcert: '/etc/ssl/certs/ca.crt' + } + + expected = 'postgresql://localhost/mydb?sslrootcert=/etc/ssl/certs/ca.crt' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with SSL client certificate without mode' do + db_config = { + adapter: 'postgresql', + host: 'localhost', + database: 'mydb', + sslcert: '/path/to/cert.crt', + sslkey: '/path/to/key.key' + } + + expected = 'postgresql://localhost/mydb?sslcert=/path/to/cert.crt&sslkey=/path/to/key.key' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with all SSL parameters (mTLS)' do + db_config = { + adapter: 'postgresql', + username: 'user', + password: 'password', + host: 'host', + port: 5432, + database: 'database', + sslmode: 'verify-full', + sslrootcert: '/ca.crt', + sslcert: '/client.crt', + sslkey: '/client.key' + } + + expected = 'postgresql://user:password@host:5432/database?sslmode=verify-full&sslrootcert=/ca.crt&sslcert=/client.crt&sslkey=/client.key' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with IPv6 host' do + db_config = { + adapter: 'postgresql', + host: '::1', + database: 'database' + } + + expected = 'postgresql://[::1]/database' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with IPv6 host and port' do + db_config = { + adapter: 'postgresql', + username: 'user', + host: '2001:db8::1234', + port: 5433, + database: 'database' + } + + expected = 'postgresql://user@[2001:db8::1234]:5433/database' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with alternative adapter name' do + db_config = { + adapter: 'postgres', + host: 'localhost', + database: 'mydb' + } + + expected = 'postgres://localhost/mydb' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'connection with database name containing underscores' do + db_config = { + adapter: 'postgresql', + username: 'postgres', + password: 'secret', + host: 'localhost', + database: 'test_db' + } + + expected = 'postgresql://postgres:secret@localhost/test_db' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'production-like configuration with SSL' do + db_config = { + adapter: 'postgresql', + username: 'app_user', + password: 'secure_password', + host: 'db.example.com', + port: 5432, + database: 'production_db', + sslmode: 'verify-full', + sslrootcert: '/etc/ssl/certs/server-ca.pem', + sslcert: '/etc/ssl/certs/client-cert.pem', + sslkey: '/etc/ssl/private/client-key.pem' + } + + expected = 'postgresql://app_user:secure_password@db.example.com:5432/production_db?sslmode=verify-full&sslrootcert=/etc/ssl/certs/server-ca.pem&sslcert=/etc/ssl/certs/client-cert.pem&sslkey=/etc/ssl/private/client-key.pem' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'OpenShift-like configuration' do + db_config = { + adapter: 'postgresql', + username: 'zync', + password: 'zync-password', + host: 'postgresql.example.svc.cluster.local', + port: 5432, + database: 'zync_production', + sslmode: 'require', + sslrootcert: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + } + + expected = 'postgresql://zync:zync-password@postgresql.example.svc.cluster.local:5432/zync_production?sslmode=require&sslrootcert=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + # Unix socket connections + test 'Unix socket with default socket directory' do + db_config = { + adapter: 'postgresql', + database: 'mydb', + host: '/var/run/postgresql' + } + + expected = 'postgresql:///mydb?host=/var/run/postgresql' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'Unix socket with username' do + db_config = { + adapter: 'postgresql', + username: 'user', + database: 'mydb', + host: '/var/run/postgresql' + } + + expected = 'postgresql://user@/mydb?host=/var/run/postgresql' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'Unix socket with username and password' do + db_config = { + adapter: 'postgresql', + username: 'user', + password: 'password', + database: 'mydb', + host: '/var/run/postgresql' + } + + expected = 'postgresql://user:password@/mydb?host=/var/run/postgresql' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'Unix socket with custom socket directory' do + db_config = { + adapter: 'postgresql', + username: 'postgres', + database: 'development', + host: '/opt/postgresql/run' + } + + expected = 'postgresql://postgres@/development?host=/opt/postgresql/run' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'Unix socket with port parameter' do + db_config = { + adapter: 'postgresql', + database: 'mydb', + host: '/var/run/postgresql', + port: 5433 + } + + expected = 'postgresql:///mydb?host=/var/run/postgresql&port=5433' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'Unix socket with SSL parameters' do + db_config = { + adapter: 'postgresql', + username: 'user', + database: 'mydb', + host: '/var/run/postgresql', + sslmode: 'require' + } + + expected = 'postgresql://user@/mydb?host=/var/run/postgresql&sslmode=require' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end + + test 'Unix socket with all parameters' do + db_config = { + adapter: 'postgresql', + username: 'appuser', + password: 'secret', + database: 'production_db', + host: '/var/run/postgresql', + port: 5432, + } + + expected = 'postgresql://appuser:secret@/production_db?host=/var/run/postgresql&port=5432' + assert_equal expected, Que::DBConnectionURL.build_connection_url(db_config) + end +end From f18bb0afcc8852c35897ee2b26ba9d24f5cb9318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 27 Jan 2026 13:58:14 +0100 Subject: [PATCH 10/11] Fix tests with SSL (cherry picked from commit e5dcae26810c39e5bf657be89ed9700c878e311f) --- config/initializers/que.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/config/initializers/que.rb b/config/initializers/que.rb index b7ac080c..29bbbbfc 100644 --- a/config/initializers/que.rb +++ b/config/initializers/que.rb @@ -15,10 +15,17 @@ def Que.start! require 'que/locker' + require 'que/db_connection_url' # Workaround for https://github.com/chanks/que/pull/192 require 'active_record/base' - Que.locker = Que::Locker.new(**Rails.application.config.x.que) + + # Build connection URL with SSL parameters from database config + # Workaround for https://github.com/que-rb/que/issues/442 + db_config = ActiveRecord::Base.connection_db_config.configuration_hash + connection_url = Que::DBConnectionURL.build_connection_url(db_config) + + Que.locker = Que::Locker.new(connection_url: connection_url, **Rails.application.config.x.que) end def Que.stop! From d28cc45fc6661175ecac076c4255575d33041f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 28 Jan 2026 18:18:46 +0100 Subject: [PATCH 11/11] New job to run tests suite over SSL * Save cache after bundle install * Docker executor * Enforce only SSL for the SSL pipeline. (cherry picked from commit 0050367216b292b326aa90d606eb333891812859) --- .circleci/config.yml | 152 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac30899e..a5110de8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,44 @@ version: 2.1 + +commands: + bundle_install: + steps: + - run: + name: bundle install + command: | + bundle config set --local force_ruby_platform true + bundle config set --local deployment 'true' + bundle config set --local path 'vendor/bundle' + bundle install --jobs $(grep -c processor /proc/cpuinfo) --retry 3 + + boot_zync: + steps: + - run: + name: boot zync + command: | + BUNDLE_WITHOUT=development:test bundle exec bin/rails runner --environment=production 'puts Rails.env' + + setup_db: + steps: + - run: + name: Set up the DB + command: | + bundle exec bin/rails db:wait db:setup + + run_tests: + steps: + - run: + name: rails test + command: | + circleci tests glob "test/**/*_test.rb" | circleci tests run --command="xargs bundle exec rake test TESTOPTS='-v'" --verbose --split-by=timings + + run_license_finder: + steps: + - run: + name: license_finder + command: | + bundle exec license_finder + jobs: docker-build: resource_class: small @@ -12,6 +52,7 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: zync RAILS_ENV: production + SECRET_KEY_BASE: test steps: - checkout - setup_remote_docker @@ -20,8 +61,8 @@ jobs: - run: docker build --tag zync:build --file ./Dockerfile . - run: command: | - docker run --net net0 -e RAILS_ENV=${RAILS_ENV} -e DATABASE_URL=${DATABASE_URL} \ - zync:build rails db:setup + docker run --net net0 -e RAILS_ENV=${RAILS_ENV} -e DATABASE_URL=${DATABASE_URL} -e SECRET_KEY_BASE=${SECRET_KEY_BASE} \ + zync:build bundle exec rails db:setup build: parameters: @@ -36,51 +77,98 @@ jobs: RAILS_ENV: test DISABLE_SPRING: 1 # we can't really run spring as it hangs on local circleci build DATABASE_URL: postgres://postgres:@localhost/circle_test + SECRET_KEY_BASE: test steps: - checkout - - # Restore bundle cache - restore_cache: keys: - zync-bundle-v2-{{ .Environment.CACHE_VERSION }}-{{ arch }}-{{ checksum "Gemfile.lock" }} - - - run: - name: bundle install - command: | - gem install bundler --version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tr -d ' '| tail -n 1) --no-document - bundle config --local force_ruby_platform true - bundle config set --local deployment 'true' - bundle config set --local path 'vendor/bundle' - bundle install --jobs $(grep -c processor /proc/cpuinfo) --retry 3 - - run: - name: boot zync - command: BUNDLE_WITHOUT=development:test bundle exec bin/rails runner --environment=production 'puts Rails.env' - + - bundle_install - save_cache: key: zync-bundle-v2-{{ .Environment.CACHE_VERSION }}-{{ arch }}-{{ checksum "Gemfile.lock" }} paths: - vendor/bundle + - boot_zync + - setup_db + - run_tests + - run_license_finder + - store_test_results: + path: test/reports + build_ssl: + parameters: + postgresql_image: + type: string + working_directory: /opt/app-root/zync + docker: + - image: quay.io/3scale/zync:ci-builder + - image: << parameters.postgresql_image >> + command: + - bash + - -c + - | + openssl req -nodes -new -x509 -subj "/CN=localhost" -keyout /server.key -out /server.crt -days 365 2>/dev/null + chown postgres:postgres /server.key /server.crt + chmod 600 /server.key + # Configure pg_hba.conf to only allow SSL connections (hostssl instead of host) + # local rule is needed for postgres internal initialization via unix socket + echo "local all all trust" > /tmp/pg_hba.conf + echo "hostssl all all all trust" >> /tmp/pg_hba.conf + exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/server.crt -c ssl_key_file=/server.key -c hba_file=/tmp/pg_hba.conf + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: circle_test + environment: + RAILS_ENV: test + DISABLE_SPRING: 1 + DATABASE_URL: postgres://postgres:@localhost/circle_test + DATABASE_SSL_MODE: verify-full + SECRET_KEY_BASE: test + steps: + - checkout - run: - name: Set up the DB - command: bundle exec bin/rails db:wait db:setup - + name: Wait for PostgreSQL to be ready + command: | + for i in $(seq 1 30); do + if openssl s_client -starttls postgres -connect localhost:5432 &1 | grep -q "SSL handshake"; then + echo "PostgreSQL SSL is ready" + exit 0 + fi + echo "Waiting for PostgreSQL SSL... ($i/30)" + sleep 1 + done + echo "PostgreSQL failed to start with SSL" + exit 1 - run: - name: rails test + name: Extract server CA certificate and generate client certificate command: | - circleci tests glob "test/**/*_test.rb" | circleci tests run --command="xargs bundle exec rake test TESTOPTS='-v'" --verbose --split-by=timings + mkdir -p .ssl + # Extract the server certificate (CA) using openssl s_client + openssl s_client -starttls postgres -connect localhost:5432 -showcerts /dev/null \ + | openssl x509 -outform PEM > .ssl/ca.crt + # Generate client key and certificate (self-signed, PostgreSQL doesn't verify client certs by default) + openssl req -nodes -new -x509 -subj "/CN=postgres" -keyout .ssl/client.key -out .ssl/client.crt -days 365 2>/dev/null + chmod 600 .ssl/client.key - run: - name: license_finder + name: Set SSL environment variables command: | - bundle exec license_finder - - - store_test_results: - path: test/reports - + echo 'export DATABASE_SSL_CA=/opt/app-root/zync/.ssl/ca.crt' >> "$BASH_ENV" + echo 'export DATABASE_SSL_CERT=/opt/app-root/zync/.ssl/client.crt' >> "$BASH_ENV" + echo 'export DATABASE_SSL_KEY=/opt/app-root/zync/.ssl/client.key' >> "$BASH_ENV" + - restore_cache: + keys: + - zync-bundle-v2-{{ .Environment.CACHE_VERSION }}-{{ arch }}-{{ checksum "Gemfile.lock" }} + - bundle_install - save_cache: - key: zync-branch-v2-{{ arch }}-{{ .Branch }} + key: zync-bundle-v2-{{ .Environment.CACHE_VERSION }}-{{ arch }}-{{ checksum "Gemfile.lock" }} paths: - vendor/bundle + - boot_zync + - setup_db + - run_tests + - run_license_finder + - store_test_results: + path: test/reports workflows: version: 2.1 @@ -89,5 +177,9 @@ workflows: - build: matrix: parameters: - postgresql_image: [ "cimg/postgres:10.22", "cimg/postgres:12.15", "cimg/postgres:13.11", "cimg/postgres:14.12" ] + postgresql_image: [ "cimg/postgres:14.19", "cimg/postgres:15.14", "cimg/postgres:16.10", "cimg/postgres:17.6", "cimg/postgres:18.0" ] + - build_ssl: + matrix: + parameters: + postgresql_image: [ "cimg/postgres:14.19", "cimg/postgres:15.14", "cimg/postgres:16.10", "cimg/postgres:17.6", "cimg/postgres:18.0" ] - docker-build