Updated: Mon Mar 5 15:43:11 2018
model name	: Intel(R) Atom(TM) CPU  C2750  @ 2.40GHz
Processors: 8
RAM MiB: 15990

Cache type: disk_cache

This benchmarks NOT reusing the same template instances between iterations, but where the template file does not change between iterations. This is somewhat silly.

Function HTMLMason HTMLTemplate HTMLTemplateCompiled MojoTemplate TemplateSandbox TemplateToolkit Tenjin TextClevery TextMicroTemplate TextTemplate TextTmpl TextXslate
complex_variable_expression 186/s n/a n/a n/a 327/s 232/s 736/s 319/s n/a n/a n/a 356/s
constant_expression 226/s n/a n/a n/a 1694/s 515/s 1062/s 854/s n/a n/a n/a 1199/s
constant_function 216/s n/a n/a n/a 1625/s n/a 875/s n/a n/a n/a n/a 454/s
records_loop_template 140/s 82.3/s 92.6/s n/a 153/s 50.1/s 375/s 322/s n/a n/a n/a 389/s
variable_expression 211/s n/a n/a n/a 599/s 361/s 1050/s 579/s n/a n/a n/a 754/s
variable_function 208/s n/a n/a n/a 502/s 341/s 893/s n/a n/a n/a n/a 481/s
variable_if_else_template 164/s 811/s 200/s n/a 488/s 283/s 583/s 478/s n/a n/a n/a 585/s

Cache type: instance_reuse

This benchmarks the most common scenario on webapps: the template engine is reused, and the templates are likely in the disk cache already.

Function HTMLMason HTMLTemplate HTMLTemplateCompiled MojoTemplate TemplateSandbox TemplateToolkit Tenjin TextClevery TextMicroTemplate TextTemplate TextTmpl TextXslate
complex_variable_expression 1758/s n/a n/a n/a 904/s 932/s 1145/s 19266/s 7612/s 339/s n/a 21804/s
constant_expression 2270/s n/a n/a n/a 20223/s 4699/s 1402/s 41718/s 18145/s 510/s n/a 54580/s
constant_function 2111/s n/a n/a n/a 18225/s n/a 1170/s n/a 16239/s 451/s n/a 9657/s
records_loop_template 1465/s n/a n/a n/a 219/s 136/s 441/s 8523/s 3380/s n/a n/a 9078/s
variable_expression 2221/s n/a n/a n/a 2473/s 2011/s 1556/s 27695/s 15759/s 457/s n/a 34021/s
variable_function 2105/s n/a n/a n/a 1505/s 1310/s 1217/s n/a 15406/s 429/s n/a 11664/s
variable_if_else_template 2226/s n/a n/a n/a 2432/s 1934/s 794/s 28752/s 14749/s n/a n/a 35855/s

Cache type: uncached_disk

This benchmarks the "first deploy" of a new template file, i.e. when the new file is not yet present on the disk cache. This seldom happens.

Function HTMLMason HTMLTemplate HTMLTemplateCompiled MojoTemplate TemplateSandbox TemplateToolkit Tenjin TextClevery TextMicroTemplate TextTemplate TextTmpl TextXslate
complex_variable_expression 86.8/s n/a n/a 111/s 38.0/s n/a 265/s 12.3/s 166/s 230/s n/a 12.6/s
constant_expression 111/s n/a n/a 233/s 181/s n/a 481/s 24.6/s 252/s 387/s n/a 26.7/s
constant_function 105/s n/a n/a 197/s 93.4/s n/a 376/s n/a 236/s 328/s n/a 19.4/s
records_loop_template 47.4/s 78.3/s 16.1/s 77.3/s 53.6/s n/a 123/s 9.55/s 102/s n/a 929/s 10.5/s
variable_expression 102/s n/a n/a 168/s 111/s n/a 396/s 25.0/s 212/s 324/s n/a 26.2/s
variable_function 103/s n/a n/a 183/s 93.9/s n/a 388/s n/a 220/s 317/s n/a 19.2/s
variable_if_else_template 47.7/s 167/s 18.0/s 71.5/s 76.4/s n/a 111/s 15.0/s 107/s n/a 1437/s 13.9/s

Benchmarking script, benchmark.sh:
#!/bin/bash

set -eou pipefail

for plugin in TextXslate TemplateToolkit Tenjin TemplateSandbox TextClevery TextMicroTemplate TextTemplate HTMLMason MojoTemplate HTMLTemplate HTMLTemplateCompiled TextCaml TextTemplateLite TextTmpl; do
	for cache_type in instance_reuse disk_cache uncached_disk; do
		for feature in variable_function constant_function complex_variable_expression variable_expression variable_if_else_template records_loop_template constant_expression; do
			out="$plugin.$cache_type.$feature.json"
			if [ -s $out ]; then
				>&2 echo "$out already exists. Skipping."
			else
				>&2 date
				>&2 echo "Benchmarking $out"
				benchmark_template_engines --duration=30 --onlyplugin $plugin --nofeatures --notypes --$cache_type --$feature --json > $out
			fi
			perl summarise.pl > templates_benchmark.html
		done
	done
done

perl summarise.pl > templates_benchmark.html

Summary script, summarise.pl:
#!/usr/bin/env perl
use 5.020_000;
use warnings;
use Cpanel::JSON::XS qw<>;
use Path::Tiny qw<path>;

my $JSON = Cpanel::JSON::XS->new->utf8;
my @jsons = glob '*.*.*.json';
my %results;
my %cache_types;
my %functions;
my %templates;
for my $file (@jsons) { # TextTemplate.disk_cache.constant_function.json
	my ($template, $cache_type, $function, $json) = split /[.]/xms, $file;
	my $data = eval { $JSON->decode(path($file)->slurp_utf8) };
	next
		if !$data
		|| ref $data ne 'HASH'
		|| !exists $data->{result} 
		|| $data->{result} ne 'SUCCESS'
		|| !exists $data->{benchmarks}
		|| !exists $data->{benchmarks}[0]{comparison}[1][1]
		;
	# For TT, I'm interested in the TT_XCET version, not pure-perl TT
	# nor TT_X (TT + XS stash only)
	my $val = $data->{benchmarks}[0]{comparison}[1][1];
	if ($template eq 'TemplateToolkit') {
		my $comparisons = $data->{benchmarks}[0]{comparison};
		($val) = map { $_->[1] } grep { $_->[0] eq 'TT_XCET' } @$comparisons;
	}
	$val =~ s!/s\z!!xms;
	$results{ $cache_type }{ $function }{ $template } = $val;
	$cache_types{ $cache_type }++;
	$functions{ $function }++; $templates{ $template }++;
}

say '<!DOCTYPE html>';
say '<html lang="en">';
say '<head>';
say '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">';
say '<meta name=viewport content="width=device-width,initial-scale=1.0">';
say '<title>Templates benchmark ';
say scalar localtime;
say '</title>';
say '<style>';
say '.highest { background-color: #dfd }';
say '.lowest  { background-color: #fdd }';
say '.important { background-color: #eee }';
say '</style>';
say '</head>';
say '<body>';

say 'Updated: ', scalar localtime;

# Describe the machine the tests were ran on
say '<pre>';
print qx!grep 'model name' /proc/cpuinfo | head -n 1!;
print 'Processors: ', qx!grep 'model name' /proc/cpuinfo | wc -l!;
print 'RAM MiB: ', qx!free -m | awk '{ if (/Mem/) { print \$2 } }'!;
say '</pre>';

# "Overall"
my @functions = sort keys %functions;
my @templates = sort keys %templates;
for my $cache_type (sort keys %cache_types) {
	say "<h1>Cache type: $cache_type</h1>";
	if ($cache_type eq 'uncached_disk') {
		say '<p>This benchmarks the "first deploy" of a new template file, i.e. when the new file is not yet present on the disk cache. This seldom happens.</p>';
	} elsif ($cache_type eq 'disk_cache') {
		say '<p>This benchmarks NOT reusing the same template instances between iterations, but where the template file does not change between iterations. This is somewhat silly.</p>';
	} elsif ($cache_type eq 'instance_reuse') {
		say '<p>This benchmarks the most common scenario on webapps: the template engine is reused, and the templates are likely in the disk cache already.</p>';
	}
	say '<table>';
	say '<tr>';
	say '<th>Function</th>';
	say "<th>$_</th>" for @templates;
	say '</tr>';
	for my $function (sort keys %functions) {
		if ($function eq 'complex_variable_expression' || $function eq 'records_loop_template') {
			say '<tr class=important>';
		} else {
			say '<tr>';
		}
		say "<th>$function</th>";
		my @sorted = sort { $results{ $cache_type }{ $function }{ $a } <=> $results{ $cache_type }{ $function }{ $b } } keys %{ $results{ $cache_type }{ $function } };
		for my $template (@templates) {
			if ($template eq $sorted[0]) {
				say '<td class=lowest>';
			} elsif ($template eq $sorted[-1]) {
				say '<td class=highest>';
			} else {
				say '<td>';
			}
			if (exists $results{ $cache_type }{ $function }{ $template }) {
				say "$results{$cache_type}{$function}{$template}/s";
			} else {
				say 'n/a';
			}
			say '</td>';
		}
		say '</tr>';
	}
	say '</table>';
}

say '<hr>';
say 'Benchmarking script, <code>benchmark.sh</code>:';
say '<pre>', html_escape(path('benchmark.sh')->slurp_utf8), '</pre>';
say '<hr>';
say 'Summary script, <code>summarise.pl</code>:';
say '<pre>', html_escape(path($0)->slurp_utf8), '</pre>';

say '</body>';
say '</html>';
exit 0;

sub html_escape {
	my $data = shift;
	$data =~ s!&!&amp;!xmsg;
	$data =~ s!<!&lt;!xmsg;
	$data =~ s!>!&gt;!xmsg;
	$data =~ s!"!&quot;!xmsg;
	return $data;
}