Implement FIPS functions on OpenSSL 3.

This commit is to implement the `OpenSSL::OPENSSL_FIPS`, `ossl_fips_mode_get`
and `ossl_fips_mode_set` to pass the test `test/openssl/test_fips.rb`.

It seems that the `OPENSSL_FIPS` macro is not used on the FIPS mode case any
more, and some FIPS related APIs also were removed in OpenSSL 3.

See the document <https://github.com/openssl/openssl/blob/master/doc/man7/migration_guide.pod#removed-fips_mode-and-fips_mode_set>
the section OPENSSL 3.0 > Main Changes from OpenSSL 1.1.1 >
Other notable deprecations and changes - Removed FIPS_mode() and FIPS_mode_set() .

The `OpenSSL::OPENSSL_FIPS` returns always true in OpenSSL 3 because the used
functions `EVP_default_properties_enable_fips` and `EVP_default_properties_is_fips_enabled`
works with the OpenSSL installed without FIPS option.

The `TEST_RUBY_OPENSSL_FIPS_ENABLED` is set on the FIPS mode case on the CI.
Because I want to test that the `OpenSSL.fips_mode` returns the `true` or
'false' surely in the CI. You can test the FIPS mode case by setting
`TEST_RUBY_OPENSSL_FIPS_ENABLED` on local too. Right now I don't find a better
way to get the status of the FIPS mode enabled or disabled for this purpose. I
am afraid of the possibility that the FIPS test case is unintentionally skipped.

I also replaced the ambiguous "returns" with "should return" in the tests.

Author: junaruga

ext/openssl/ossl.c

@@ -418,7 +418,11 @@ static VALUE
 ossl_fips_mode_get(VALUE self)
 {
 
-#ifdef OPENSSL_FIPS
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+    VALUE enabled;
+    enabled = EVP_default_properties_is_fips_enabled(NULL) ? Qtrue : Qfalse;
+    return enabled;
+#elif OPENSSL_FIPS
     VALUE enabled;
     enabled = FIPS_mode() ? Qtrue : Qfalse;
     return enabled;
@@ -442,8 +446,18 @@ ossl_fips_mode_get(VALUE self)
 static VALUE
 ossl_fips_mode_set(VALUE self, VALUE enabled)
 {
-
-#ifdef OPENSSL_FIPS
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+    if (RTEST(enabled)) {
+        if (!EVP_default_properties_enable_fips(NULL, 1)) {
+            ossl_raise(eOSSLError, "Turning on FIPS mode failed");
+        }
+    } else {
+        if (!EVP_default_properties_enable_fips(NULL, 0)) {
+            ossl_raise(eOSSLError, "Turning off FIPS mode failed");
+        }
+    }
+    return enabled;
+#elif OPENSSL_FIPS
     if (RTEST(enabled)) {
 	int mode = FIPS_mode();
 	if(!mode && !FIPS_mode_set(1)) /* turning on twice leads to an error */
@@ -1198,7 +1212,10 @@ Init_openssl(void)
      * Boolean indicating whether OpenSSL is FIPS-capable or not
      */
     rb_define_const(mOSSL, "OPENSSL_FIPS",
-#ifdef OPENSSL_FIPS
+/* OpenSSL 3 is FIPS-capable even when it is installed without fips option */
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+                    Qtrue
+#elif OPENSSL_FIPS
 		    Qtrue
 #else
 		    Qfalse

test/openssl/test_fips.rb

@@ -4,20 +4,44 @@
 if defined?(OpenSSL)
 
 class OpenSSL::TestFIPS < OpenSSL::TestCase
+  def test_fips_mode_get_is_true_on_fips_mode_enabled
+    unless ENV["TEST_RUBY_OPENSSL_FIPS_ENABLED"]
+      omit "Only for FIPS mode environment"
+    end
+
+    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+      assert OpenSSL.fips_mode == true, ".fips_mode should return true on FIPS mode enabled"
+    end;
+  end
+
+  def test_fips_mode_get_is_false_on_fips_mode_disabled
+    if ENV["TEST_RUBY_OPENSSL_FIPS_ENABLED"]
+      omit "Only for non-FIPS mode environment"
+    end
+
+    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+      message = ".fips_mode should return false on FIPS mode disabled. " \
+                "If you run the test on FIPS mode, please set " \
+                "TEST_RUBY_OPENSSL_FIPS_ENABLED=true"
+      assert OpenSSL.fips_mode == false, message
+    end;
+  end
+
   def test_fips_mode_is_reentrant
     OpenSSL.fips_mode = false
     OpenSSL.fips_mode = false
   end
 
-  def test_fips_mode_get
-    return unless OpenSSL::OPENSSL_FIPS
+  def test_fips_mode_get_with_fips_mode_set
+    omit('OpenSSL is not FIPS-capable') unless OpenSSL::OPENSSL_FIPS
+
     assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
       begin
         OpenSSL.fips_mode = true
-        assert OpenSSL.fips_mode == true, ".fips_mode returns true when .fips_mode=true"
+        assert OpenSSL.fips_mode == true, ".fips_mode should return true when .fips_mode=true"
 
         OpenSSL.fips_mode = false
-        assert OpenSSL.fips_mode == false, ".fips_mode returns false when .fips_mode=false"
+        assert OpenSSL.fips_mode == false, ".fips_mode should return false when .fips_mode=false"
       rescue OpenSSL::OpenSSLError
         pend "Could not set FIPS mode (OpenSSL::OpenSSLError: \#$!); skipping"
       end