diff --git a/.rubocop.yml b/.rubocop.yml
index 73760a1b..bf71d6db 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -1,2 +1,11 @@
 inherit_from:
   - https://relaxed.ruby.style/rubocop.yml
+
+AllCops:
+  Exclude:
+    - tmp/**/*
+
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - "**/bin/*"
+    - "**/exe/*"
diff --git a/bin/setup b/bin/setup
index dce67d86..b4f06885 100755
--- a/bin/setup
+++ b/bin/setup
@@ -1,8 +1,20 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
 
-bundle install
+GEM_ROOT = File.expand_path('..', __dir__)
 
-# Do any other automated setup that you need to do here
+def system(*args)
+  puts "$ #{args.size == 1 ? args.first : args.shelljoin}"
+  super
+end
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+cd GEM_ROOT
+
+puts "\n== Installing Ruby dependencies =="
+system! %{gem install bundler --conservative}
+system! %{bundle check || bundle install}
diff --git a/bin/solidus b/bin/solidus
new file mode 100755
index 00000000..0cef6dd1
--- /dev/null
+++ b/bin/solidus
@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+load "#{__dir__}/../exe/solidus"
diff --git a/exe/solidus b/exe/solidus
index 8c66bef3..f5171b51 100755
--- a/exe/solidus
+++ b/exe/solidus
@@ -3,8 +3,7 @@
 
 require 'optparse'
 
-require 'solidus_dev_support'
-require 'solidus_dev_support/extension'
+require 'solidus_dev_support/version'
 
 Options = Struct.new(:name)
 
@@ -46,6 +45,8 @@ if ARGV.first == 'extension' || ARGV.first == 'e'
     exit 1
   end
 
+  require 'solidus_dev_support'
+  require 'solidus_dev_support/extension'
   SolidusDevSupport::Extension.start
 else
   Parser.parse(ARGV)
diff --git a/lib/solidus_dev_support/extension.rb b/lib/solidus_dev_support/extension.rb
index d1efeed2..211ee217 100644
--- a/lib/solidus_dev_support/extension.rb
+++ b/lib/solidus_dev_support/extension.rb
@@ -2,6 +2,7 @@
 
 require 'thor'
 require 'thor/group'
+require 'pathname'
 
 module SolidusDevSupport
   class Extension < Thor::Group
@@ -23,6 +24,10 @@ def generate
       directory '.circleci', "#{file_name}/.circleci"
       directory '.github', "#{file_name}/.github"
 
+      Dir["#{file_name}/bin/*"].each do |executable|
+        make_executable executable
+      end
+
       template 'extension.gemspec.erb', "#{file_name}/#{file_name}.gemspec"
       template 'Gemfile', "#{file_name}/Gemfile"
       template 'gitignore', "#{file_name}/.gitignore"
@@ -45,6 +50,12 @@ def class_name
       def use_prefix(prefix)
         @file_name = prefix + Thor::Util.snake_case(file_name) unless file_name =~ /^#{prefix}/
       end
+
+      def make_executable(path)
+        path = Pathname(path)
+        executable = (path.stat.mode | 0o111)
+        path.chmod(executable)
+      end
     end
   end
 end
diff --git a/lib/solidus_dev_support/templates/extension/bin/console.tt b/lib/solidus_dev_support/templates/extension/bin/console.tt
new file mode 100755
index 00000000..03c56be4
--- /dev/null
+++ b/lib/solidus_dev_support/templates/extension/bin/console.tt
@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "<%= file_name %>"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+$LOAD_PATH.unshift(*Dir["#{__dir__}/../app/*"])
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)
diff --git a/lib/solidus_dev_support/templates/extension/bin/rails b/lib/solidus_dev_support/templates/extension/bin/rails
new file mode 100755
index 00000000..cfe59976
--- /dev/null
+++ b/lib/solidus_dev_support/templates/extension/bin/rails
@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+app_root = 'spec/dummy'
+
+unless File.exist? "#{app_root}/bin/rails"
+  system "bin/rake", app_root or begin # rubocop:disable Style/AndOr
+    warn "Automatic creation of the dummy app failed"
+    exit 1
+  end
+end
+
+Dir.chdir 'spec/dummy'
+exec 'bin/rails', *ARGV
diff --git a/lib/solidus_dev_support/templates/extension/bin/rails.tt b/lib/solidus_dev_support/templates/extension/bin/rails.tt
deleted file mode 100644
index 6f97dd98..00000000
--- a/lib/solidus_dev_support/templates/extension/bin/rails.tt
+++ /dev/null
@@ -1,7 +0,0 @@
-# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
-
-ENGINE_ROOT = File.expand_path('../..', __FILE__)
-ENGINE_PATH = File.expand_path('../../lib/<%= file_name -%>/engine', __FILE__)
-
-require 'rails/all'
-require 'rails/engine/commands'
diff --git a/lib/solidus_dev_support/templates/extension/bin/setup b/lib/solidus_dev_support/templates/extension/bin/setup
new file mode 100755
index 00000000..e97c371a
--- /dev/null
+++ b/lib/solidus_dev_support/templates/extension/bin/setup
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+gem install bundler --conservative
+bundle update
+bin/rake clobber
+bin/rake spec/dummy
+bin/rails railties:install:migrations db:migrate db:seed