From 1001b315a3780a0b3a37c2331f0fe58c6be65a0b Mon Sep 17 00:00:00 2001 From: Nate Hardison Date: Thu, 30 May 2013 15:48:55 -0700 Subject: [PATCH 1/4] Preprocess assets as Django management command Rather than directly invoke command-line Python (and Mako) from the assets Rakefile, or call an external Python script, use a Django management command to preprocess all asset template files. An "asset template file" is defined as a static asset file with a file extension indicating that it needs to be run through a template engine prior to Sass/CoffeeScript compilation or packaging with other assets. The preprocess_assets management command will look through all of the files listed in the `STATICFILES_DIRS`, preprocessing each as needed. Preprocessing strips off the special template file extension, creating a new file in the process. Currently, the only variable accessible in an asset template file is the `THEME_NAME`, defined in the settings. --- cms/envs/common.py | 1 + .../mitxmako/management/__init__.py | 0 .../mitxmako/management/commands/__init__.py | 0 .../management/commands/preprocess_assets.py | 65 +++++++++++++++++++ lms/envs/common.py | 1 + rakefiles/assets.rake | 57 ++++++---------- 6 files changed, 88 insertions(+), 36 deletions(-) create mode 100644 common/djangoapps/mitxmako/management/__init__.py create mode 100644 common/djangoapps/mitxmako/management/commands/__init__.py create mode 100644 common/djangoapps/mitxmako/management/commands/preprocess_assets.py diff --git a/cms/envs/common.py b/cms/envs/common.py index ed905727159c..04d5888750e4 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -323,6 +323,7 @@ 'track', # For asset pipelining + 'mitxmako', 'pipeline', 'staticfiles', 'static_replace', diff --git a/common/djangoapps/mitxmako/management/__init__.py b/common/djangoapps/mitxmako/management/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/common/djangoapps/mitxmako/management/commands/__init__.py b/common/djangoapps/mitxmako/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/common/djangoapps/mitxmako/management/commands/preprocess_assets.py b/common/djangoapps/mitxmako/management/commands/preprocess_assets.py new file mode 100644 index 000000000000..36a2da9ad3c6 --- /dev/null +++ b/common/djangoapps/mitxmako/management/commands/preprocess_assets.py @@ -0,0 +1,65 @@ +""" +Preprocess templatized asset files, enabling asset authors to use +Python/Django inside of Sass and CoffeeScript. This preprocessing +will happen before the invocation of the asset compiler (currently +handled by the asset Rakefile). + +For this to work, assets need to be named with the appropriate +template extension (e.g., .mako for Mako templates). Currently Mako +is the only template engine supported. +""" +import os + +from django.core.management.base import NoArgsCommand +from django.conf import settings + +from mako.template import Template + +class Command(NoArgsCommand): + """ + Basic management command to preprocess asset template files. + """ + + help = "Preprocess asset template files to ready them for compilation." + + def handle_noargs(self, **options): + """ + Walk over all of the static files directories specified in the + settings file, looking for asset template files (indicated by + a file extension like .mako). + """ + for staticfiles_dir in getattr(settings, "STATICFILES_DIRS", []): + # Cribbed from the django-staticfiles app at: + # https://github.com/jezdez/django-staticfiles/blob/develop/staticfiles/finders.py#L52 + if isinstance(staticfiles_dir, (list, tuple)): + prefix, staticfiles_dir = staticfiles_dir + + # Walk over the current static files directory tree, + # preprocessing files that have a template extension. + for root, dirs, files in os.walk(staticfiles_dir): + for filename in files: + outfile, extension = os.path.splitext(filename) + # We currently only handle Mako templates + if extension == ".mako": + self.__preprocess(os.path.join(root, filename), + os.path.join(root, outfile)) + + + def __context(self): + """ + Return a dict that contains all of the available context + variables to the asset template. + """ + # TODO: do we need to include anything else? + # TODO: do this with the django-settings-context-processor + return { "THEME_NAME" : getattr(settings, "THEME_NAME", None) } + + + def __preprocess(self, infile, outfile): + """ + Run `infile` through the Mako template engine, storing the + result in `outfile`. + """ + with open(outfile, "w") as _outfile: + _outfile.write(Template(filename=str(infile)).render(env=self.__context())) + diff --git a/lms/envs/common.py b/lms/envs/common.py index f75dcf8804e3..410287972321 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -662,6 +662,7 @@ 'service_status', # For asset pipelining + 'mitxmako', 'pipeline', 'staticfiles', 'static_replace', diff --git a/rakefiles/assets.rake b/rakefiles/assets.rake index 68127a317f4f..aca6ea2e5245 100644 --- a/rakefiles/assets.rake +++ b/rakefiles/assets.rake @@ -6,30 +6,6 @@ if USE_CUSTOM_THEME THEME_SASS = File.join(THEME_ROOT, "static", "sass") end -# Run the specified file through the Mako templating engine, providing -# the ENV_TOKENS to the templating context. -def preprocess_with_mako(filename) - # simple command-line invocation of Mako engine - # cdodge: the .gsub() are used to translate true->True and false->False to make the generated - # python actually valid python. This is just a short term hack to unblock the release train - # until a real fix can be made by people who know this better - mako = "from mako.template import Template;" + - "print Template(filename=\"#{filename}\")" + - # Total hack. It works because a Python dict literal has - # the same format as a JSON object. - ".render(env=#{ENV_TOKENS.to_json.gsub("true","True").gsub("false","False")});" - - # strip off the .mako extension - output_filename = filename.chomp(File.extname(filename)) - - # just pipe from stdout into the new file, exiting on failure - File.open(output_filename, 'w') do |file| - file.write(`python -c '#{mako}'`) - exit_code = $?.to_i - abort "#{mako} failed with #{exit_code}" if exit_code.to_i != 0 - end -end - def xmodule_cmd(watch=false, debug=false) xmodule_cmd = 'xmodule_assets common/static/xmodule' if watch @@ -84,11 +60,12 @@ namespace :assets do desc "Compile all assets in debug mode" multitask :debug - desc "Preprocess all static assets that have the .mako extension" - task :preprocess do - # Run assets through the Mako templating engine. Right now we - # just hardcode the asset filenames. - preprocess_with_mako("lms/static/sass/application.scss.mako") + desc "Preprocess all templatized static asset files" + task :preprocess, [:system, :env] do |t, args| + args.with_defaults(:system => "lms", :env => "dev") + sh(django_admin(args.system, args.env, "preprocess_assets")) do |ok, status| + abort "asset preprocessing failed!" if !ok + end end desc "Watch all assets for changes and automatically recompile" @@ -138,7 +115,6 @@ namespace :assets do end end - multitask :sass => 'assets:xmodule' namespace :sass do # In watch mode, sass doesn't immediately compile out of date files, @@ -153,16 +129,25 @@ namespace :assets do end end +# This task does the real heavy lifting to gather all of the static +# assets. We want people to call it via the wrapper below, so we +# don't provide a description so that it won't show up in rake -T. +task :gather_assets, [:system, :env] => :assets do |t, args| + sh("#{django_admin(args.system, args.env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status| + if !ok + abort "collectstatic failed!" + end + end +end + [:lms, :cms].each do |system| # Per environment tasks environments(system).each do |env| + # This task wraps the one above, since we need the system and + # env arguments to be passed to all dependent tasks. desc "Compile coffeescript and sass, and then run collectstatic in the specified environment" - task "#{system}:gather_assets:#{env}" => :assets do - sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status| - if !ok - abort "collectstatic failed!" - end - end + task "#{system}:gather_assets:#{env}" do + task(:gather_assets).invoke(system, env) end end end From 35fb5fa47ed3817e9dd6cfea7adaf173d08c15da Mon Sep 17 00:00:00 2001 From: Nate Hardison Date: Sun, 2 Jun 2013 15:34:00 -0700 Subject: [PATCH 2/4] Add wrapper task to start LMS/CMS server The LMS/CMS server startup depends on the :assets task, which needs to receive :system and :env arguments. Since the existing server startup tasks didn't take a :system argument, a new command called :runserver is created that does take :system, :env, and :options arguments. The old server startup tasks are adjusted to wrap the one :runserver task that does all of the heavy lifting. --- rakefiles/django.rake | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rakefiles/django.rake b/rakefiles/django.rake index 8b42192130c8..910d4e39cc92 100644 --- a/rakefiles/django.rake +++ b/rakefiles/django.rake @@ -15,14 +15,22 @@ task :fastlms do sh("#{django_admin} runserver --traceback --settings=lms.envs.dev --pythonpath=.") end +# Start :system locally with the specified :env and :options. +# +# This task should be invoked via the wrapper below, so we don't +# include a description to keep it from showing up in rake -T. +task :runserver, [:system, :env, :options] => [:install_prereqs, 'assets:_watch', :predjango] do |t, args| + sh(django_admin(args.system, args.env, 'runserver', args.options)) +end + [:lms, :cms].each do |system| desc <<-desc Start the #{system} locally with the specified environment (defaults to dev). Other useful environments are devplus (for dev testing with a real local database) desc - task system, [:env, :options] => [:install_prereqs, 'assets:_watch', :predjango] do |t, args| + task system, [:env, :options] do |t, args| args.with_defaults(:env => 'dev', :options => default_options[system]) - sh(django_admin(system, args.env, 'runserver', args.options)) + task(:runserver).invoke(system, args.env, args.options) end desc "Start #{system} Celery worker" From 1fd1f63075f2db8cb3a47e98a54b2870090599b8 Mon Sep 17 00:00:00 2001 From: Nate Hardison Date: Sun, 2 Jun 2013 15:36:59 -0700 Subject: [PATCH 3/4] Wrapper tasks for :assets-dependent Jasmine tasks The :browse_jasmine_ and :phantomjs_jasmine_ tasks depend on the :assets task, which needs to receive both :system and :env arguments. Therefore, new tasks (:browse_jasmine and :phantomjs_jasmine) are created that do take :system and :env args. The old :browse_jasmine_ and :phantomjs_jasmine_ tasks now wrap the new tasks, passing in as an argument and 'jasmine' (for :env, since it's hardcoded to 'jasmine' in django_for_jasmine()). --- rakefiles/jasmine.rake | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/rakefiles/jasmine.rake b/rakefiles/jasmine.rake index 4182bef9e250..ecf88eb4e5fc 100644 --- a/rakefiles/jasmine.rake +++ b/rakefiles/jasmine.rake @@ -73,21 +73,43 @@ def run_phantom_js(url) sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}") end +# Open jasmine tests for :system in the default browser. The :env +# should (always?) be 'jasmine', but it's passed as an arg so that +# the :assets dependency gets it. +# +# This task should be invoked via the wrapper below, so we don't +# include a description to keep it from showing up in rake -T. +task :browse_jasmine, [:system, :env] => :assets do |t, args| + django_for_jasmine(args.system, true) do |jasmine_url| + Launchy.open(jasmine_url) + puts "Press ENTER to terminate".red + $stdin.gets + end +end + +# Use phantomjs to run jasmine tests from the console. The :env +# should (always?) be 'jasmine', but it's passed as an arg so that +# the :assets dependency gets it. +# +# This task should be invoked via the wrapper below, so we don't +# include a description to keep it from showing up in rake -T. +task :phantomjs_jasmine, [:system, :env] => :assets do |t, args| + django_for_jasmine(args.system, false) do |jasmine_url| + run_phantom_js(jasmine_url) + end +end + +# Wrapper tasks for the real browse_jasmine and phantomjs_jasmine +# tasks above. These have a nicer UI since there's no arg passing. [:lms, :cms].each do |system| desc "Open jasmine tests for #{system} in your default browser" - task "browse_jasmine_#{system}" => :assets do - django_for_jasmine(system, true) do |jasmine_url| - Launchy.open(jasmine_url) - puts "Press ENTER to terminate".red - $stdin.gets - end + task "browse_jasmine_#{system}" do + task(:browse_jasmine).invoke(system, 'jasmine') end desc "Use phantomjs to run jasmine tests for #{system} from the console" - task "phantomjs_jasmine_#{system}" => :assets do - django_for_jasmine(system, false) do |jasmine_url| - run_phantom_js(jasmine_url) - end + task "phantomjs_jasmine_#{system}" do + task(:phantomjs_jasmine).invoke(system, 'jasmine') end end From 03a9765b79f91ecde0dc9ac4591ddef055da62dc Mon Sep 17 00:00:00 2001 From: Nate Hardison Date: Mon, 3 Jun 2013 10:00:50 -0700 Subject: [PATCH 4/4] Change `task.invoke` to `Rake::Task.invoke` Using `task()` reopens the definition of the task, and all we really need to do is get a reference to the task itself to invoke it. --- rakefiles/assets.rake | 2 +- rakefiles/django.rake | 2 +- rakefiles/jasmine.rake | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rakefiles/assets.rake b/rakefiles/assets.rake index aca6ea2e5245..45a7de1102ef 100644 --- a/rakefiles/assets.rake +++ b/rakefiles/assets.rake @@ -147,7 +147,7 @@ end # env arguments to be passed to all dependent tasks. desc "Compile coffeescript and sass, and then run collectstatic in the specified environment" task "#{system}:gather_assets:#{env}" do - task(:gather_assets).invoke(system, env) + Rake::Task[:gather_assets].invoke(system, env) end end end diff --git a/rakefiles/django.rake b/rakefiles/django.rake index 910d4e39cc92..b1adf24050fd 100644 --- a/rakefiles/django.rake +++ b/rakefiles/django.rake @@ -30,7 +30,7 @@ end desc task system, [:env, :options] do |t, args| args.with_defaults(:env => 'dev', :options => default_options[system]) - task(:runserver).invoke(system, args.env, args.options) + Rake::Task[:runserver].invoke(system, args.env, args.options) end desc "Start #{system} Celery worker" diff --git a/rakefiles/jasmine.rake b/rakefiles/jasmine.rake index ecf88eb4e5fc..bd1c7e5d6c77 100644 --- a/rakefiles/jasmine.rake +++ b/rakefiles/jasmine.rake @@ -104,12 +104,12 @@ end [:lms, :cms].each do |system| desc "Open jasmine tests for #{system} in your default browser" task "browse_jasmine_#{system}" do - task(:browse_jasmine).invoke(system, 'jasmine') + Rake::Task[:browse_jasmine].invoke(system, 'jasmine') end desc "Use phantomjs to run jasmine tests for #{system} from the console" task "phantomjs_jasmine_#{system}" do - task(:phantomjs_jasmine).invoke(system, 'jasmine') + Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine') end end