diff --git a/internal/tools/securitypolicy/README.md b/internal/tools/securitypolicy/README.md
index f4c4738b43..20c75b1535 100644
--- a/internal/tools/securitypolicy/README.md
+++ b/internal/tools/securitypolicy/README.md
@@ -114,13 +114,26 @@ TOML configuration file to process (required)
 
 output raw JSON in addition to the Base64 encoded version
 
-- -u
+## Authorization
 
-username to use to login to remote container services (defaults to anonymous)
+Some images will be pulled from registries that require authorization. To add
+authorization information for a given image, you would add an `[auth]` object
+to the TOML definiton for that image. For example:
 
-- -p
+```toml
+[[image]]
+name = "rust:1.52.1"
+command = ["rustc", "--help"]
+
+[auth]
+username = "my username"
+password = "my password"
+```
+
+Authorization information needs added on a per-image basis as it can vary from
+image to image and their respective registries.
 
-password to use to login to remote container services (defaults to anonymous)
+To pull an image using anonymous access, no `[auth]` object is required.
 
 ## Pause container
 
diff --git a/internal/tools/securitypolicy/main.go b/internal/tools/securitypolicy/main.go
index 3aba848601..591b2ac7d0 100644
--- a/internal/tools/securitypolicy/main.go
+++ b/internal/tools/securitypolicy/main.go
@@ -22,8 +22,6 @@ import (
 var (
 	configFile = flag.String("c", "", "config")
 	outputJSON = flag.Bool("j", false, "json")
-	username   = flag.String("u", "", "username")
-	password   = flag.String("p", "", "password")
 )
 
 func main() {
@@ -87,10 +85,16 @@ type EnvironmentVariableRule struct {
 
 type Image struct {
 	Name     string                    `toml:"name"`
+	Auth     ImageAuth                 `toml:"auth"`
 	Command  []string                  `toml:"command"`
 	EnvRules []EnvironmentVariableRule `toml:"env_rule"`
 }
 
+type ImageAuth struct {
+	Username string `toml:"username"`
+	Password string `toml:"password"`
+}
+
 type Config struct {
 	AllowAll bool    `toml:"allow_all"`
 	Images   []Image `toml:"image"`
@@ -107,16 +111,6 @@ func createPolicyFromConfig(config Config) (sp.SecurityPolicy, error) {
 		Containers: map[string]sp.SecurityPolicyContainer{},
 	}
 
-	var imageOptions []remote.Option
-	if len(*username) != 0 && len(*password) != 0 {
-		auth := authn.Basic{
-			Username: *username,
-			Password: *password}
-		c, _ := auth.Authorization()
-		authOption := remote.WithAuth(authn.FromConfig(*c))
-		imageOptions = append(imageOptions, authOption)
-	}
-
 	// Hardcode the pause container version and command. We still pull it
 	// to get the root hash and any environment variable rules we might need.
 	pause := Image{
@@ -126,6 +120,17 @@ func createPolicyFromConfig(config Config) (sp.SecurityPolicy, error) {
 	config.Images = append(config.Images, pause)
 
 	for _, image := range config.Images {
+		var imageOptions []remote.Option
+
+		if image.Auth.Username != "" && image.Auth.Password != "" {
+			auth := authn.Basic{
+				Username: image.Auth.Username,
+				Password: image.Auth.Password}
+			c, _ := auth.Authorization()
+			authOption := remote.WithAuth(authn.FromConfig(*c))
+			imageOptions = append(imageOptions, authOption)
+		}
+
 		// validate EnvRules
 		err := validateEnvRules(image.EnvRules)
 		if err != nil {