Initial project import

tools/v/vlib/veb/assets/README.md

@@ -0,0 +1,177 @@
+# Assets
+
+The asset manager for veb. You can use this asset manager to minify CSS and JavaScript files,
+combine them into a single file and to make sure the asset you're using exists.
+
+## Usage
+
+Add `AssetManager` to your App struct to use the asset manager.
+
+**Example:**
+
+```v
+module main
+
+import veb
+import veb.assets
+
+pub struct Context {
+	veb.Context
+}
+
+pub struct App {
+pub mut:
+	am assets.AssetManager
+}
+
+fn main() {
+	mut app := &App{}
+	veb.run[App, Context](mut app, 8080)
+}
+```
+
+### Including assets
+
+If you want to include an asset in your templates you can use the `include` method.
+First pass the type of asset (css or js), then specify the "include name" of an asset.
+
+**Example:**
+
+```html
+@{app.am.include(.css, 'main.css')}
+```
+
+Will generate
+
+```html
+<link rel="stylesheet" href="/main.css" />
+```
+
+### Adding assets
+
+To add an asset use the `add` method. You must specify the path of the asset and what its
+include name will be: the name that you will use in templates.
+
+**Example:**
+
+```v ignore
+// add a css file at the path "css/main.css" and set its include name to "main.css"
+app.am.add(.css, 'css/main.css', 'main.css')
+```
+
+### Minify and Combine assets
+
+If you want to minify each asset you must set the `minify` field and specify the cache
+folder. Each assest you add is minifed and outputted in `cache_dir`.
+
+**Example:**
+
+```v ignore
+pub struct App {
+pub mut:
+	am assets.AssetManager = assets.AssetManager{
+		cache_dir: 'dist'
+		minify: true
+	}
+}
+```
+
+To combine the all currently added assets into a single file you must call the `combine` method
+and specify which asset type you want to combine.
+
+**Example:**
+
+```v ignore
+// `combine` returns the path of the minified file
+minified_file := app.am.combine(.css)!
+```
+
+### Handle folders
+
+You can use the asset manger in combination with veb's `StaticHandler` to serve
+assets in a folder as static assets.
+
+**Example:**
+
+```v ignore
+pub struct App {
+	veb.StaticHandler
+pub mut:
+	am assets.AssetManager
+}
+```
+
+Let's say we have the following folder structure:
+
+```
+assets/
+├── css/
+│   └── main.css
+└── js/
+	└── main.js
+```
+
+We can tell the asset manager to add all assets in the `static` folder
+
+**Example:**
+
+```v ignore
+fn main() {
+	mut app := &App{}
+	// add all assets in the "assets" folder
+	app.am.handle_assets('assets')!
+	// serve all files in the "assets" folder as static files
+	app.handle_static('assets', false)!
+	// start the app
+	veb.run[App, Context](mut app, 8080)
+}
+```
+
+The include name of each minified asset will be set to its relative path,
+so if you want to include `main.css` in your template you would write
+`@{app.am.include('css/main.css')}`
+
+#### Minify
+
+If you add an asset folder and want to minify those assets you can call the
+`cleanup_cache` method to remove old files from the cache folder
+that are no longer needed.
+
+**Example:**
+
+```v ignore
+pub struct App {
+	veb.StaticHandler
+pub mut:
+	am assets.AssetManager = assets.AssetManager{
+		cache_dir: 'dist'
+		minify: true
+	}
+}
+
+fn main() {
+	mut app := &App{}
+	// add all assets in the "assets" folder
+	app.am.handle_assets('assets')!
+	// remove all old cached files from the cache folder
+	app.am.cleanup_cache()!
+	// serve all files in the "assets" folder as static files
+	app.handle_static('assets', false)!
+	// start the app
+	veb.run[App, Context](mut app, 8080)
+}
+```
+
+#### Prefix the include name
+
+You can add a custom prefix to the include name of assets when adding a folder.
+
+**Example:**
+
+```v ignore
+// add all assets in the "assets" folder
+app.am.handle_assets_at('assets', 'static')!
+```
+
+Now if you want to include `main.css` you would write
+``@{app.am.include('static/css/main.css')}`

tools/v/vlib/veb/assets/assets.v

@@ -0,0 +1,312 @@
+module assets
+
+import crypto.md5
+import os
+import strings
+import time
+import veb
+
+pub enum AssetType {
+	css
+	js
+	all
+}
+
+pub struct Asset {
+pub:
+	kind          AssetType
+	file_path     string
+	last_modified time.Time
+	include_name  string
+}
+
+pub struct AssetManager {
+mut:
+	css               []Asset
+	js                []Asset
+	cached_file_names []string
+pub mut:
+	// when true assets will be minified
+	minify bool
+	// the directory to store the cached/combined files
+	cache_dir string
+	// how a combined file should be named. For example for css the extension '.css'
+	// will be added to the end of `combined_file_name`
+	combined_file_name string = 'combined'
+}
+
+fn (mut am AssetManager) add_asset_directory(directory_path string, traversed_path string) ! {
+	files := os.ls(directory_path)!
+	if files.len > 0 {
+		for file in files {
+			full_path := os.join_path(directory_path, file)
+			relative_path := os.join_path(traversed_path, file)
+
+			if os.is_dir(full_path) {
+				am.add_asset_directory(full_path, relative_path)!
+			} else {
+				ext := os.file_ext(full_path)
+				match ext {
+					'.css' { am.add(.css, full_path, relative_path)! }
+					'.js' { am.add(.js, full_path, relative_path)! }
+					// ignore non css/js files
+					else {}
+				}
+			}
+		}
+	}
+}
+
+// handle_assets recursively walks `directory_path` and adds any assets to the asset manager
+pub fn (mut am AssetManager) handle_assets(directory_path string) ! {
+	return am.add_asset_directory(directory_path, '')
+}
+
+// handle_assets_at recursively walks `directory_path` and adds any assets to the asset manager.
+// The include name of assets are prefixed with `prepend`
+pub fn (mut am AssetManager) handle_assets_at(directory_path string, prepend string) ! {
+	// remove trailing '/'
+	return am.add_asset_directory(directory_path, prepend.trim_right('/'))
+}
+
+// get all assets of type `asset_type`
+pub fn (am AssetManager) get_assets(asset_type AssetType) []Asset {
+	return match asset_type {
+		.css {
+			am.css
+		}
+		.js {
+			am.js
+		}
+		.all {
+			mut assets := []Asset{}
+			assets << am.css
+			assets << am.js
+			assets
+		}
+	}
+}
+
+// add an asset to the asset manager
+pub fn (mut am AssetManager) add(asset_type AssetType, file_path string, include_name string) ! {
+	if asset_type == .all {
+		return error('cannot minify asset of type "all"')
+	}
+	if !os.exists(file_path) {
+		return error('cannot add asset: file "${file_path}" does not exist')
+	}
+
+	last_modified_unix := os.file_last_mod_unix(file_path)
+
+	mut real_path := file_path
+
+	if am.minify {
+		// minify and cache file if it was modified
+		output_path, is_cached := am.minify_and_cache(asset_type, real_path, last_modified_unix,
+			include_name)!
+
+		if is_cached == false && am.exists(asset_type, include_name) {
+			// file was not modified between the last call to `add`
+			// and the file was already in the asset manager, so we don't need to
+			// add it again
+			return
+		}
+
+		real_path = output_path
+	}
+
+	asset := Asset{
+		kind:          asset_type
+		file_path:     real_path
+		last_modified: time.unix(last_modified_unix)
+		include_name:  include_name
+	}
+
+	match asset_type {
+		.css { am.css << asset }
+		.js { am.js << asset }
+		else {}
+	}
+}
+
+fn (mut am AssetManager) minify_and_cache(asset_type AssetType, file_path string, last_modified i64, include_name string) !(string, bool) {
+	if asset_type == .all {
+		return error('cannot minify asset of type "all"')
+	}
+
+	if am.cache_dir == '' {
+		return error('cannot minify asset: cache directory is not valid')
+	} else if !os.exists(am.cache_dir) {
+		os.mkdir_all(am.cache_dir)!
+	}
+
+	cache_key := am.get_cache_key(file_path, last_modified)
+	output_file := '${cache_key}.${asset_type}'
+	output_path := os.join_path(am.cache_dir, output_file)
+
+	if os.exists(output_path) {
+		// the output path already exists, this means that the file has
+		// been minifed and cached before and hasn't changed in the meantime
+		am.cached_file_names << output_file
+		return output_path, false
+	} else {
+		// check if the file has been minified before, but is modified.
+		// if that's the case we remove the old cached file
+		cached_files := os.ls(am.cache_dir)!
+		hash := cache_key.all_before('-')
+		for file in cached_files {
+			if file.starts_with(hash) {
+				os.rm(os.join_path(am.cache_dir, file))!
+			}
+		}
+	}
+
+	txt := os.read_file(file_path)!
+	minified := match asset_type {
+		.css { minify_css(txt) }
+		.js { minify_js(txt) }
+		else { '' }
+	}
+
+	os.write_file(output_path, minified)!
+
+	am.cached_file_names << output_file
+	return output_path, true
+}
+
+fn (mut am AssetManager) get_cache_key(file_path string, last_modified i64) string {
+	abs_path := if os.is_abs_path(file_path) { file_path } else { os.resource_abs_path(file_path) }
+	hash := md5.sum(abs_path.bytes())
+	return '${hash.hex()}-${last_modified}'
+}
+
+// cleanup_cache removes all files in the cache directory that aren't cached at the time
+// this function is called
+pub fn (mut am AssetManager) cleanup_cache() ! {
+	if am.cache_dir == '' {
+		return error('[veb.assets]: cache directory is not valid')
+	}
+	cached_files := os.ls(am.cache_dir)!
+
+	// loop over all the files in the cache directory. If a file isn't cached, remove it
+	for file in cached_files {
+		ext := os.file_ext(file)
+		if ext !in ['.css', '.js'] || file in am.cached_file_names {
+			continue
+		} else if !file.starts_with(am.combined_file_name) {
+			os.rm(os.join_path(am.cache_dir, file))!
+		}
+	}
+}
+
+// check if an asset is already added to the asset manager
+pub fn (am AssetManager) exists(asset_type AssetType, include_name string) bool {
+	assets := am.get_assets(asset_type)
+
+	return assets.any(it.include_name == include_name)
+}
+
+// include css/js files in your veb app from templates
+// Usage example:
+// ```html
+// @{app.am.include(.css, 'main.css')}
+// ```
+pub fn (am AssetManager) include(asset_type AssetType, include_name string) veb.RawHtml {
+	assets := am.get_assets(asset_type)
+	for asset in assets {
+		if asset.include_name == include_name {
+			// always add link/src from root of web server ('/css/main.css'),
+			// but leave absolute paths intact
+			mut real_path := asset.file_path
+			if real_path[0] != `/` && !os.is_abs_path(real_path) {
+				real_path = '/${asset.file_path}'
+			}
+
+			return match asset_type {
+				.css {
+					'<link rel="stylesheet" href="${real_path}">'
+				}
+				.js {
+					'<script src="${real_path}"></script>'
+				}
+				else {
+					eprintln('[veb.assets] can only include css or js assets')
+					''
+				}
+			}
+		}
+	}
+	eprintln('[veb.assets] no asset with include name "${include_name}" exists!')
+	return ''
+}
+
+// combine assets of type `asset_type` into a single file and return the outputted file path.
+// If you call `combine` with asset type `all` the function will return an empty string,
+// the minified files will be available at `combined_file_name`.`asset_type`
+pub fn (mut am AssetManager) combine(asset_type AssetType) !string {
+	if asset_type == .all {
+		am.combine(.css)!
+		am.combine(.js)!
+		return ''
+	}
+	if am.cache_dir == '' {
+		return error('cannot combine assets: cache directory is not valid')
+	} else if !os.exists(am.cache_dir) {
+		os.mkdir_all(am.cache_dir)!
+	}
+
+	assets := am.get_assets(asset_type)
+	combined_file_path := os.join_path(am.cache_dir, '${am.combined_file_name}.${asset_type}')
+	mut f := os.create(combined_file_path)!
+
+	for asset in assets {
+		bytes := os.read_bytes(asset.file_path)!
+		f.write(bytes)!
+		f.write_string('\n')!
+	}
+
+	f.close()
+
+	return combined_file_path
+}
+
+// TODO: implement proper minification
+@[manualfree]
+pub fn minify_css(css string) string {
+	mut lines := css.split('\n')
+	// estimate arbitrary number of characters for a line of css
+	mut sb := strings.new_builder(lines.len * 20)
+	defer {
+		unsafe { sb.free() }
+	}
+
+	for line in lines {
+		trimmed := line.trim_space()
+		if trimmed != '' {
+			sb.write_string(trimmed)
+		}
+	}
+
+	return sb.str()
+}
+
+// TODO: implement proper minification
+@[manualfree]
+pub fn minify_js(js string) string {
+	mut lines := js.split('\n')
+	// estimate arbitrary number of characters for a line of js
+	mut sb := strings.new_builder(lines.len * 40)
+	defer {
+		unsafe { sb.free() }
+	}
+
+	for line in lines {
+		trimmed := line.trim_space()
+		if trimmed != '' {
+			sb.write_string(trimmed)
+			sb.write_u8(` `)
+		}
+	}
+
+	return sb.str()
+}

tools/v/vlib/veb/assets/assets_test.v

@@ -0,0 +1,191 @@
+import veb.assets
+import os
+
+const base_cache_dir = os.join_path(os.vtmp_dir(), 'veb_assets_test_cache')
+
+fn testsuite_begin() {
+	os.mkdir_all(base_cache_dir) or {}
+}
+
+fn testsuite_end() {
+	os.rmdir_all(base_cache_dir) or {}
+}
+
+// clean_cache_dir used before and after tests that write to a cache directory.
+// Because of parallel compilation and therefore test running,
+// unique cache dirs are needed per test function.
+fn clean_cache_dir(dir string) {
+	os.rmdir_all(dir) or {}
+}
+
+fn cache_dir(test_name string) string {
+	return os.join_path(base_cache_dir, test_name)
+}
+
+fn get_test_file_path(file string) string {
+	path := os.join_path(base_cache_dir, file)
+	os.rm(path) or {}
+	os.write_file(path, get_test_file_contents(file)) or { panic(err) }
+	return path
+}
+
+fn get_test_file_contents(file string) string {
+	contents := match file {
+		'test1.js' { '{"one": 1}\n' }
+		'test2.js' { '{"two": 2}\n' }
+		'test1.css' { '.one {\n\tcolor: #336699;\n}\n' }
+		'test2.css' { '.two {\n\tcolor: #996633;\n}\n' }
+		else { 'wibble\n' }
+	}
+
+	return contents
+}
+
+fn test_add() {
+	mut am := assets.AssetManager{}
+
+	mut errored := false
+	am.add(.css, 'test.css', 'test.css') or { errored = true }
+	assert errored == true, 'am.add should error'
+
+	errored = false
+	am.add(.css, get_test_file_path('test1.css'), 'included.css') or {
+		eprintln(err)
+		errored = true
+	}
+	assert errored == false, 'am.add should not error'
+
+	css_assets := am.get_assets(.css)
+	assert css_assets.len == 1
+	assert css_assets[0].file_path == get_test_file_path('test1.css')
+	assert css_assets[0].include_name == 'included.css'
+}
+
+fn test_add_minify_missing_cache_dir() {
+	mut am := assets.AssetManager{
+		minify: true
+	}
+	mut errored := false
+	am.add(.js, get_test_file_path('test1.css'), 'included.js') or {
+		assert err.msg() == 'cannot minify asset: cache directory is not valid'
+		errored = true
+	}
+
+	assert errored == true, 'am.add should return an error'
+}
+
+fn test_add_minified() {
+	mut am := assets.AssetManager{
+		minify:    true
+		cache_dir: cache_dir('test_add_minified')
+	}
+	clean_cache_dir(am.cache_dir)
+
+	am.add(.js, get_test_file_path('test1.js'), 'included.js')!
+
+	js_assets := am.get_assets(.js)
+	assert js_assets.len == 1
+	assert js_assets[0].file_path.starts_with(am.cache_dir) == true
+}
+
+fn test_combine() {
+	mut am := assets.AssetManager{
+		cache_dir: cache_dir('test_combine')
+	}
+	clean_cache_dir(am.cache_dir)
+
+	am.add(.css, get_test_file_path('test1.css'), 'test1.css')!
+	am.add(.css, get_test_file_path('test2.css'), 'test2.css')!
+
+	combined_path := am.combine(.css)!
+	combined := os.read_file(combined_path)!
+
+	expected := get_test_file_contents('test1.css') + '\n' + get_test_file_contents('test2.css') +
+		'\n'
+	assert combined == expected
+}
+
+fn test_combine_minified() {
+	// minify test is simple for now, because assets are not properly minified yet
+	mut am := assets.AssetManager{
+		cache_dir: cache_dir('test_combine_minified')
+		minify:    true
+	}
+	clean_cache_dir(am.cache_dir)
+
+	am.add(.css, get_test_file_path('test1.css'), 'test1.css')!
+	am.add(.css, get_test_file_path('test2.css'), 'test2.css')!
+
+	combined_path := am.combine(.css)!
+	combined := os.read_file(combined_path)!
+
+	// minified version should be 2 lines + one extra newline
+	assert combined.split('\n').len == 3
+}
+
+fn test_minify_cache_last_modified() {
+	mut am := assets.AssetManager{
+		minify:    true
+		cache_dir: cache_dir('test_cache_last_modified')
+	}
+	clean_cache_dir(am.cache_dir)
+
+	// first we write the file and add it
+	am.add(.js, get_test_file_path('test1.js'), 'included.js')!
+	mut js_assets := am.get_assets(.js)
+	assert js_assets.len == 1
+	old_cached_path := js_assets[0].file_path
+
+	// then we only add the file, the file is not modified so the "last modified is the same".
+	// we expect that the asset manager doesn't cache a minified file if it hasn't been changed
+	// the last time it was added
+	am.add(.js, os.join_path(base_cache_dir, 'test1.js'), 'included.js')!
+
+	js_assets = am.get_assets(.js)
+	// check if the file isn't added twice
+	assert js_assets.len == 1
+	// if the file path was not modified, veb.assets didn't overwrite the file
+	assert js_assets[0].file_path == old_cached_path
+}
+
+fn test_cleanup_cache() {
+	mut am := assets.AssetManager{
+		minify:    true
+		cache_dir: cache_dir('test_cleanup_cache')
+	}
+	clean_cache_dir(am.cache_dir)
+	// manually make the cache dir
+	os.mkdir_all(am.cache_dir) or {}
+
+	// write a file to the cache dir isn't added to the asset manager to represent
+	// a previously cached file
+	path1 := os.join_path(am.cache_dir, 'test1.css')
+	os.write_file(path1, 'h1 { color: red; }')!
+	assert os.exists(path1) == true
+
+	// add a file to the asset manager and write it
+	am.add(.css, get_test_file_path('test2.css'), 'test2.css')!
+	css_assets := am.get_assets(.css)
+	// get the cached path
+	assert css_assets.len == 1
+	path2 := css_assets[0].file_path
+	assert os.exists(path2) == true
+
+	am.cleanup_cache()!
+
+	// the first asset wasn't added to the asset manager, so it should not exist
+	assert os.exists(path1) == false
+	assert os.exists(path2) == true
+}
+
+fn test_include() {
+	mut am := assets.AssetManager{}
+
+	css_path := get_test_file_path('test1.css')
+	js_path := get_test_file_path('test1.js')
+	am.add(.css, css_path, 'other.css')!
+	am.add(.js, js_path, 'js/test.js')!
+
+	assert am.include(.css, 'other.css') == '<link rel="stylesheet" href="${css_path}">'
+	assert am.include(.js, 'js/test.js') == '<script src="${js_path}"></script>'
+}