Initial project import

This commit is contained in:
Agila Forge Importer committed 2026-07-13 15:20:32 +00:00
commit 23f2d29e32
11957 files changed
+1754577

No files matched your search

+31
View File
@@ -0,0 +1,31 @@
net/html is an **HTML Parser** written in pure V.
## Usage
```v
import net.html
fn main() {
doc := html.parse('<html><body><h1 class="title">Hello world!</h1></body></html>')
tag := doc.get_tags(name: 'h1')[0] // <h1>Hello world!</h1>
println(tag.name) // h1
println(tag.content) // Hello world!
println(tag.attributes) // {'class':'title'}
println(tag.str()) // <h1 class="title">Hello world!</h1>
}
```
For tags with nested markup, `tag.content` preserves the inner HTML:
```v
import net.html
fn main() {
doc := html.parse('<p>before <span>in between</span> after</p>')
tag := doc.get_tags(name: 'p')[0]
println(tag.content) // before <span>in between</span> after
println(tag.text()) // before in between after
}
```
More examples found on [`parser_test.v`](parser_test.v) and [`html_test.v`](html_test.v)
+89
View File
@@ -0,0 +1,89 @@
module html
const null_element = int(-1)
struct Stack {
mut:
elements []int
size int
}
@[inline]
fn is_null(data int) bool {
return data == null_element
}
@[inline]
fn (stack Stack) is_empty() bool {
return stack.size <= 0
}
fn (stack Stack) peek() int {
return if !stack.is_empty() { stack.elements[stack.size - 1] } else { null_element }
}
fn (mut stack Stack) pop() int {
mut to_return := null_element
if !stack.is_empty() {
to_return = stack.elements[stack.size - 1]
stack.size--
}
return to_return
}
fn (mut stack Stack) push(item int) {
if stack.elements.len > stack.size {
stack.elements[stack.size] = item
} else {
stack.elements << item
}
stack.size++
}
struct BTree {
mut:
all_tags []Tag
node_pointer int
childrens [][]int
parents []int
}
fn (mut btree BTree) add_children(tag Tag) int {
btree.all_tags << tag
if btree.all_tags.len > 1 {
for btree.childrens.len <= btree.node_pointer {
mut temp_array := btree.childrens.clone()
temp_array << []int{}
btree.childrens = temp_array
}
btree.childrens[btree.node_pointer] << btree.all_tags.len - 1
for btree.parents.len < btree.all_tags.len {
mut temp_array := btree.parents.clone()
temp_array << 0
btree.parents = temp_array
}
btree.parents[btree.all_tags.len - 1] = btree.node_pointer
}
return btree.all_tags.len - 1
}
@[inline]
fn (btree BTree) get_children() []int {
return btree.childrens[btree.node_pointer]
}
@[inline]
fn (btree BTree) get_parent() int {
return btree.parents[btree.node_pointer]
}
@[inline]
fn (btree BTree) get_stored() Tag {
return btree.all_tags[btree.node_pointer]
}
fn (mut btree BTree) move_pointer(to int) {
if to < btree.all_tags.len {
btree.node_pointer = to
}
}
+231
View File
@@ -0,0 +1,231 @@
module html
import os
import strings
// The W3C Document Object Model (DOM) is a platform and language-neutral
// interface that allows programs and scripts to dynamically access and
// update the content, structure, and style of a document.
//
// https://www.w3.org/TR/WD-DOM/introduction.html
pub struct DocumentObjectModel {
mut:
root &Tag = unsafe { nil }
constructed bool
btree BTree
all_tags []&Tag
all_attributes map[string][]&Tag
close_tags map[string]bool // add a counter to see count how many times is closed and parse correctly
attributes map[string][]string
tag_attributes map[string][][]&Tag
tag_type map[string][]&Tag
debug_file os.File
}
@[params]
pub struct GetTagsOptions {
pub:
name string
}
@[if debug_html ?]
fn (mut dom DocumentObjectModel) print_debug(data string) {
if data.len > 0 {
dom.debug_file.writeln(data) or { eprintln(err) }
}
}
@[inline]
fn is_close_tag(tag &Tag) bool {
return tag.name != '' && tag.name[0] == `/`
}
fn (mut dom DocumentObjectModel) where_is(item_name string, attribute_name string) int {
if attribute_name !in dom.attributes {
dom.attributes[attribute_name] = []string{}
}
mut string_array := dom.attributes[attribute_name]
mut counter := 0
for value in string_array {
if value == item_name {
return counter
}
counter++
}
string_array << item_name
dom.attributes[attribute_name] = string_array
return string_array.len - 1
}
fn (mut dom DocumentObjectModel) add_tag_attribute(tag &Tag) {
for attribute_name, _ in tag.attributes {
attribute_value := tag.attributes[attribute_name]
location := dom.where_is(attribute_value, attribute_name)
if attribute_name !in dom.tag_attributes {
dom.tag_attributes[attribute_name] = []
}
for {
mut temp_array := unsafe { dom.tag_attributes[attribute_name] }
temp_array << []&Tag{}
dom.tag_attributes[attribute_name] = temp_array
if location < unsafe { dom.tag_attributes[attribute_name].len } + 1 {
break
}
}
mut temp_array := unsafe { dom.tag_attributes[attribute_name][location] }
temp_array << tag
dom.tag_attributes[attribute_name][location] = temp_array
}
}
fn (mut dom DocumentObjectModel) add_tag_by_type(tag &Tag) {
tag_name := tag.name
if tag_name !in dom.tag_type {
dom.tag_type[tag_name] = [tag]
} else {
mut temp_array := unsafe { dom.tag_type[tag_name] }
temp_array << tag
dom.tag_type[tag_name] = temp_array
}
}
fn (mut dom DocumentObjectModel) add_tag_by_attribute(tag &Tag) {
for attribute_name in tag.attributes.keys() {
if attribute_name !in dom.all_attributes {
dom.all_attributes[attribute_name] = [tag]
} else {
mut temp_array := unsafe { dom.all_attributes[attribute_name] }
temp_array << tag
dom.all_attributes[attribute_name] = temp_array
}
}
}
fn (mut dom DocumentObjectModel) construct(tag_list []&Tag) {
dom.constructed = true
// If there are no tags, accessing `tag_list` below does panic.
if tag_list.len == 0 {
return
}
mut temp_map := map[string]int{}
mut temp_int := null_element
mut temp_string := ''
mut stack := Stack{}
dom.btree = BTree{}
dom.root = tag_list[0]
dom.all_tags = [tag_list[0]]
temp_map['0'] = dom.btree.add_children(tag_list[0])
stack.push(0)
root_index := 0
for index := 1; index < tag_list.len; index++ {
mut tag := tag_list[index]
dom.print_debug(tag.str())
if is_close_tag(tag) {
temp_int = stack.peek()
temp_string = tag.name[1..]
old_stack_size := stack.size
for !is_null(temp_int) && temp_string != tag_list[temp_int].name
&& !tag_list[temp_int].closed {
dom.print_debug(temp_string + ' >> ' + tag_list[temp_int].name + ' ' +
(temp_string == tag_list[temp_int].name).str())
stack.pop()
temp_int = stack.peek()
}
if is_null(temp_int) || temp_string != tag_list[temp_int].name {
stack.size = old_stack_size
continue
}
temp_int = stack.peek()
temp_int = if !is_null(temp_int) { stack.pop() } else { root_index }
if is_null(temp_int) {
stack.push(root_index)
}
dom.print_debug('Removed ' + temp_string + ' -- ' + tag_list[temp_int].name)
} else if tag.name != '' {
dom.add_tag_attribute(tag) // error here
dom.add_tag_by_attribute(tag)
dom.add_tag_by_type(tag)
dom.all_tags << tag
temp_int = stack.peek()
if !is_null(temp_int) {
dom.btree.move_pointer(temp_map[temp_int.str()])
temp_map[index.str()] = dom.btree.add_children(tag)
mut temp_tag := tag_list[temp_int]
position_in_parent := temp_tag.add_child(tag) // tag_list[temp_int] = temp_tag
tag.add_parent(temp_tag, position_in_parent)
/*
dom.print_debug("Added ${tag.name} as child of '" + tag_list[temp_int].name +
"' which now has ${dom.btree.get_children().len} childrens")
*/
dom.print_debug("Added ${tag.name} as child of '" + temp_tag.name +
"' which now has ${temp_tag.children.len} childrens")
} else { // dom.new_root(tag)
stack.push(root_index)
}
temp_string = '/' + tag.name
if temp_string in dom.close_tags && !tag.closed { // if tag ends with />
dom.print_debug('Pushed ' + temp_string)
stack.push(index)
}
}
} // println(tag_list[root_index]) for debug purposes
dom.root = tag_list[0]
mut root := dom.root
dom.normalize_tag_content(mut root)
}
fn (mut dom DocumentObjectModel) normalize_tag_content(mut tag Tag) {
tag.text_content = tag.content
tag.content_is_inner_html = true
if tag.children.len == 0 {
return
}
mut inner_html := strings.new_builder(tag.content.len + tag.children.len * 32)
inner_html.write_string(tag.text_content)
for idx := 0; idx < tag.children.len; idx++ {
mut child := tag.children[idx]
dom.normalize_tag_content(mut child)
inner_html.write_string(child.str())
}
tag.content = inner_html.str()
}
// get_root returns the root of the document.
pub fn (dom &DocumentObjectModel) get_root() &Tag {
return dom.root
}
// get_tags returns all tags stored in the document.
pub fn (dom &DocumentObjectModel) get_tags(options GetTagsOptions) []&Tag {
if options.name != '' {
return if options.name in dom.tag_type {
unsafe { dom.tag_type[options.name] }
} else {
[]&Tag{}
}
}
return dom.all_tags
}
// get_tags_by_class_name retrieves all tags recursively in the document root that have the given class name(s).
pub fn (dom &DocumentObjectModel) get_tags_by_class_name(names ...string) []&Tag {
return dom.root.get_tags_by_class_name(...names)
}
// get_tags_by_attribute retrieves all tags in the document that have the given attribute name.
pub fn (dom &DocumentObjectModel) get_tags_by_attribute(name string) []&Tag {
return if name in dom.all_attributes { unsafe { dom.all_attributes[name] } } else { []&Tag{} }
}
// get_tags_by_attribute_value retrieves all tags in the document that have the given attribute name and value.
pub fn (mut dom DocumentObjectModel) get_tags_by_attribute_value(name string, value string) []&Tag {
location := dom.where_is(value, name)
attributes := unsafe { dom.tag_attributes[name] }
if attributes.len > location {
return attributes[location]
}
return []
}
+96
View File
@@ -0,0 +1,96 @@
module html
import strings
fn generate_temp_html() string {
mut temp_html := strings.new_builder(200)
temp_html.write_string('<!doctype html><html><head><title>Giant String</title></head><body>')
for counter := 0; counter < 4; counter++ {
temp_html.write_string("<div id='name_${counter}' ")
temp_html.write_string("class='several-${counter}'>Look at ${counter}</div>")
}
temp_html.write_string('</body></html>')
return temp_html.str()
}
fn test_search_by_tag_type() {
dom := parse(generate_temp_html())
assert dom.get_tags(name: 'div').len == 4
assert dom.get_tags(name: 'head').len == 1
assert dom.get_tags(name: 'body').len == 1
}
fn test_search_by_attribute_value() {
mut dom := parse(generate_temp_html())
// println(temp_html)
print('Amount ')
println(dom.get_tags_by_attribute_value('id', 'name_0'))
assert dom.get_tags_by_attribute_value('id', 'name_0').len == 1
}
fn test_access_parent() {
mut dom := parse(generate_temp_html())
div_tags := dom.get_tags(name: 'div')
parent := div_tags[0].parent
assert unsafe { parent != 0 }
for div_tag in div_tags {
assert div_tag.parent == parent
}
}
fn test_search_by_attributes() {
dom := parse(generate_temp_html())
assert dom.get_tags_by_attribute('id').len == 4
}
fn test_tags_used() {
dom := parse(generate_temp_html())
assert dom.get_tags().len == 9
}
fn test_access_tag_fields() {
dom := parse(generate_temp_html())
id_tags := dom.get_tags_by_attribute('id')
assert id_tags[0].name == 'div'
assert id_tags[1].attributes['class'] == 'several-1'
}
fn generate_temp_html_with_classes() string {
mut temp_html := strings.new_builder(400)
temp_html.write_string('<!doctype html><html><head><title>Giant String</title></head><body>')
temp_html.write_string("<div class='single'>Single</div>")
for counter := 0; counter < 4; counter++ {
temp_html.write_string("<div id='name_${counter}' ")
temp_html.write_string("class='common'>Common No. ${counter}</div>")
}
temp_html.write_string("<div class='complex-0 complex-1 complex-2'>Complex</div>")
temp_html.write_string("<div class='complex-0 complex-2'>Partial</div>")
temp_html.write_string('</body></html>')
return temp_html.str()
}
fn test_search_by_class() {
dom := parse(generate_temp_html_with_classes())
single_class_tags := dom.get_tags_by_class_name('single')
common_class_tags := dom.get_tags_by_class_name('common')
complex_class_tags := dom.get_tags_by_class_name('complex-0', 'complex-1', 'complex-2')
partial_class_tags := dom.get_tags_by_class_name('complex-0', 'complex-2')
shuffled_class_tags := dom.get_tags_by_class_name('complex-2', 'complex-0', 'complex-1')
assert single_class_tags.len == 1
assert common_class_tags.len == 4
assert complex_class_tags.len == 1
assert complex_class_tags[0].attributes['class'] == 'complex-0 complex-1 complex-2'
assert partial_class_tags.len == 2
assert shuffled_class_tags.len == 1
assert shuffled_class_tags[0].attributes['class'] == 'complex-0 complex-1 complex-2'
}
fn test_unmatched_close_tag_is_ignored() {
content := '<!doctype html><html><body><div><a href="#">x</a></a></div><article class="news-post">hello</article></body></html>'
mut dom := parse(content)
by_attr := dom.get_tags_by_attribute_value('class', 'news-post')
by_class := dom.get_tags_by_class_name('news-post')
assert by_attr.len == 1
assert by_class.len == 1
assert by_class[0].name == 'article'
}
+22
View File
@@ -0,0 +1,22 @@
module html
import os
// parse parses and returns the DOM from the given text.
// Note: this function converts tags to lowercase.
// E.g. <MyTag>content</MyTag> is parsed as <mytag>content</mytag>.
pub fn parse(text string) DocumentObjectModel {
mut parser := Parser{}
parser.parse_html(text)
return parser.get_dom()
}
// parse_file parses and returns the DOM from the contents of a file.
// Note: this function converts tags to lowercase.
// E.g. <MyTag>content</MyTag> is parsed as <mytag>content</mytag>.
pub fn parse_file(filename string) DocumentObjectModel {
content := os.read_file(filename) or { return DocumentObjectModel{
root: &Tag{}
} }
return parse(content)
}
+42
View File
@@ -0,0 +1,42 @@
module html
fn test_parse() {
doc := parse('<html><body><h1 class="title">Hello world!</h1></body></html>')
tags := doc.get_tags(name: 'h1')
assert tags.len == 1
h1_tag := tags[0] // <h1>Hello world!</h1>
assert h1_tag.name == 'h1'
assert h1_tag.content == 'Hello world!'
assert h1_tag.attributes.len == 2
// TODO: do not remove. Attributes must not have an empty attr.
// assert h1_tag.attributes.len == 1
assert h1_tag.str() == '<h1 class="title" >Hello world!</h1>'
// assert h1_tag.str() == '<h1 class="title">Hello world!</h1>'
}
fn test_parse_inline_tags() {
doc := parse('<html><body><p>before <span>in between</span> after</p></body></html>')
tags := doc.get_tags(name: 'span')
assert tags.len == 1
span_tag := tags[0]
assert span_tag.str() == '<span>in between</span>'
p_tags := doc.get_tags(name: 'p')
assert p_tags.len == 1
p_tag := p_tags[0]
assert p_tag.content == 'before <span>in between</span> after'
assert p_tag.str() == '<p>before <span>in between</span> after</p>'
assert p_tag.text() == 'before in between after'
}
fn test_parse_content_with_nested_tags() {
doc := parse('<tr id="line"><td><a href="#">wom<nobr>All</a></td></tr>')
td_tag := doc.get_tags(name: 'td')[0]
a_tag := td_tag.get_tags('a')[0]
assert td_tag.content == '<a href="#" >wom<nobr>All</nobr></a>'
assert a_tag.content == 'wom<nobr>All</nobr>'
assert td_tag.text() == 'womAll'
}
+293
View File
@@ -0,0 +1,293 @@
module html
import os
import strings
struct LexicalAttributes {
mut:
current_tag &Tag = unsafe { nil }
open_tag bool
open_code bool
open_string int
open_comment bool
is_attribute bool
opened_code_type string
line_count int
outside_tag bool
text_after_tag bool
lexeme_builder strings.Builder = strings.new_builder(100)
code_tags map[string]bool = {
'script': true
'style': true
}
}
// Parser is responsible for reading the HTML strings and converting them into a `DocumentObjectModel`.
pub struct Parser {
mut:
dom DocumentObjectModel
lexical_attributes LexicalAttributes = LexicalAttributes{
current_tag: &Tag{}
}
filename string = 'direct-parse'
initialized bool
tags []&Tag
debug_file os.File
}
// This function is used to add a tag for the parser ignore it's content.
// For example, if you have an html or XML with a custom tag, like `<script>`, using this function,
// like `add_code_tag('script')` will make all `script` tags content be jumped,
// so you still have its content, but will not confuse the parser with it's `>` or `<`.
pub fn (mut parser Parser) add_code_tag(name string) {
if name.len <= 0 {
return
}
parser.lexical_attributes.code_tags[name] = true
}
@[inline]
fn (parser Parser) builder_str() string {
return parser.lexical_attributes.lexeme_builder.after(0)
}
@[if debug_html ?]
fn (mut parser Parser) print_debug(data string) {
if data.len > 0 {
parser.debug_file.writeln(data) or { panic(err) }
}
}
fn (mut parser Parser) verify_end_comment(remove bool) bool {
lexeme := parser.builder_str()
last := lexeme[lexeme.len - 1]
penultimate := lexeme[lexeme.len - 2]
is_end_comment := last == `-` && penultimate == `-`
if is_end_comment && remove {
parser.lexical_attributes.lexeme_builder.go_back(2)
}
return is_end_comment
}
fn blank_string(data string) bool {
mut count := 0
for chr in data {
if chr == 10 || chr == 9 || chr == 32 {
count++
}
}
return count == data.len
}
// init initializes the parser.
fn (mut parser Parser) init() {
if parser.initialized {
return
}
parser.dom = DocumentObjectModel{
debug_file: parser.debug_file
root: &Tag{}
}
parser.add_code_tag('')
parser.tags = []&Tag{}
parser.dom.close_tags['/!document'] = true
parser.lexical_attributes.current_tag = &Tag{}
parser.lexical_attributes.outside_tag = true
parser.initialized = true
}
fn (mut parser Parser) generate_tag() {
if parser.lexical_attributes.open_tag {
return
}
if parser.lexical_attributes.current_tag.name != ''
|| parser.lexical_attributes.current_tag.content.len > 0 {
parser.tags << parser.lexical_attributes.current_tag
}
parser.lexical_attributes.current_tag = &Tag{}
}
// split_parse parses the HTML fragment
pub fn (mut parser Parser) split_parse(data string) {
parser.init()
for chr in data {
is_quote := chr == `"` || chr == `'`
string_code := match chr {
`"` { 1 }
`'` { 2 }
else { 0 }
}
if parser.lexical_attributes.open_code { // verify if open_code is complete and handle string code
parser.lexical_attributes.lexeme_builder.write_u8(chr)
if parser.lexical_attributes.open_string > 0
&& parser.lexical_attributes.open_string == string_code {
parser.lexical_attributes.open_string = 0
} else if chr == `>` { // code tag is finished
name_close_tag := '</${parser.lexical_attributes.opened_code_type}>'
if parser.builder_str().to_lower().ends_with(name_close_tag) {
parser.lexical_attributes.open_code = false
// modify lexeme_builder to include script text as content in the next loop
parser.lexical_attributes.lexeme_builder.go_back(name_close_tag.len)
parser.lexical_attributes.current_tag.closed = true
parser.lexical_attributes.current_tag.close_type = .new_tag
}
}
} else if parser.lexical_attributes.open_comment {
if chr == `>` && parser.verify_end_comment(false) {
// parser.print_debug(parser.builder_str() + " >> " + parser.lexical_attributes.line_count.str())
parser.lexical_attributes.lexeme_builder.go_back_to(0)
parser.lexical_attributes.open_comment = false
parser.lexical_attributes.open_tag = false
} else {
parser.lexical_attributes.lexeme_builder.write_u8(chr)
}
} else if parser.lexical_attributes.open_string > 0 {
if parser.lexical_attributes.open_string == string_code {
parser.lexical_attributes.open_string = 0
parser.lexical_attributes.lexeme_builder.write_u8(chr)
temp_lexeme := parser.builder_str()
if parser.lexical_attributes.current_tag.last_attribute != '' {
lattr := parser.lexical_attributes.current_tag.last_attribute
nval := temp_lexeme.substr(1, temp_lexeme.len - 1)
// parser.print_debug(lattr + " = " + temp_lexeme)
parser.lexical_attributes.current_tag.attributes[lattr] = nval
if lattr == 'class' {
for class_name in nval.split_any('\t\r\n \x0D') {
if class_name != '' {
parser.lexical_attributes.current_tag.class_set.add(class_name)
}
}
}
parser.lexical_attributes.current_tag.last_attribute = ''
} else {
parser.lexical_attributes.current_tag.attributes[temp_lexeme.to_lower()] = ''
// parser.print_debug(temp_lexeme)
}
parser.lexical_attributes.lexeme_builder.go_back_to(0)
} else {
parser.lexical_attributes.lexeme_builder.write_u8(chr)
}
} else if parser.lexical_attributes.open_tag {
if parser.lexical_attributes.lexeme_builder.len == 0 && is_quote {
parser.lexical_attributes.open_string = string_code
parser.lexical_attributes.lexeme_builder.write_u8(chr)
} else if chr == `>` {
complete_lexeme := parser.builder_str().to_lower()
parser.lexical_attributes.current_tag.closed = (complete_lexeme.len > 0
&& complete_lexeme[complete_lexeme.len - 1] == `/`)
if complete_lexeme.len > 0 && complete_lexeme[0] == `/` {
parser.dom.close_tags[complete_lexeme] = true
}
/*
else if complete_lexeme.len > 0 && complete_lexeme[complete_lexeme.len - 1] == 47 { // if end tag like "/>"
parser.lexical_attributes.current_tag.closed = true
}
*/
if parser.lexical_attributes.current_tag.name == '' {
parser.lexical_attributes.current_tag.name = complete_lexeme
} else if complete_lexeme != '/' {
parser.lexical_attributes.current_tag.attributes[complete_lexeme] = ''
}
parser.lexical_attributes.open_tag = false
parser.lexical_attributes.lexeme_builder.go_back_to(0) // if tag name is code
if parser.lexical_attributes.current_tag.name in parser.lexical_attributes.code_tags {
parser.lexical_attributes.open_code = true
parser.lexical_attributes.opened_code_type = parser.lexical_attributes.current_tag.name
}
// parser.print_debug(parser.lexical_attributes.current_tag.name)
} else if chr !in [u8(9), ` `, `=`, `\n`] { // Tab, space, = and \n
parser.lexical_attributes.lexeme_builder.write_u8(chr)
} else if chr != 10 {
complete_lexeme := parser.builder_str().to_lower()
if parser.lexical_attributes.current_tag.name == '' {
parser.lexical_attributes.current_tag.name = complete_lexeme
} else {
parser.lexical_attributes.current_tag.attributes[complete_lexeme] = ''
parser.lexical_attributes.current_tag.last_attribute = ''
if chr == `=` {
parser.lexical_attributes.current_tag.last_attribute = complete_lexeme
}
}
parser.lexical_attributes.lexeme_builder.go_back_to(0)
}
if parser.builder_str() == '!--' {
parser.lexical_attributes.open_comment = true
}
} else if chr == `<` {
temp_string := parser.builder_str()
if parser.lexical_attributes.lexeme_builder.len >= 1 {
if parser.lexical_attributes.current_tag.name.len > 1
&& parser.lexical_attributes.current_tag.name[0] == 47
&& !blank_string(temp_string) {
parser.lexical_attributes.text_after_tag = true
} else {
parser.lexical_attributes.current_tag.content = temp_string // verify later who has this content
}
}
// parser.print_debug(parser.lexical_attributes.current_tag.str())
parser.lexical_attributes.lexeme_builder.go_back_to(0)
parser.generate_tag()
parser.lexical_attributes.open_tag = true
parser.lexical_attributes.outside_tag = false
if parser.lexical_attributes.text_after_tag == true {
parser.tags << &Tag{
name: 'text'
content: temp_string
}
parser.lexical_attributes.text_after_tag = false
}
} else {
parser.lexical_attributes.lexeme_builder.write_u8(chr)
}
}
// If `data` has not tags but has only text.
if parser.lexical_attributes.outside_tag {
temp_string := parser.lexical_attributes.lexeme_builder.str()
if parser.tags.len == 0 {
parser.tags << &Tag{
name: 'text'
content: temp_string
}
} else if parser.tags.len == 1 {
mut tag := parser.tags.first()
if tag.name == 'text' {
tag.content += temp_string
}
}
}
}
// parse_html parses the given HTML string
pub fn (mut parser Parser) parse_html(data string) {
parser.init()
mut lines := data.split_into_lines()
for index, line in lines {
parser.lexical_attributes.line_count++
// Parser shouldn't replace `\n`, because it may break JS code or text which sticks together.
// After `split_into_lines()` we need to add `\n` again.
parser.split_parse(if index < lines.len - 1 { '${line}\n' } else { line })
}
parser.generate_tag()
parser.dom.debug_file = parser.debug_file
parser.dom.construct(parser.tags)
}
// finalize finishes the parsing stage .
@[inline]
pub fn (mut parser Parser) finalize() {
parser.generate_tag()
}
// get_dom returns the parser's current DOM representation.
pub fn (mut parser Parser) get_dom() DocumentObjectModel {
if !parser.dom.constructed {
parser.generate_tag()
parser.dom.construct(parser.tags)
}
return parser.dom
}
+69
View File
@@ -0,0 +1,69 @@
module html
import strings
fn test_parse_empty_string() {
mut parser := Parser{}
parser.parse_html('')
assert parser.tags.len == 0
}
fn test_parse_text() {
mut parser := Parser{}
text_content := 'test\nparse\ntext'
parser.parse_html(text_content)
assert parser.tags.len == 1
assert parser.tags.first().text() == text_content
}
fn test_parse_one_tag_with_text() {
mut parser := Parser{}
text_content := 'tag\nwith\ntext'
p_tag := '<p>${text_content}</p>'
parser.parse_html(p_tag)
assert parser.tags.first().text() == text_content
}
fn test_split_parse() {
mut parser := Parser{}
parser.init()
parser.split_parse('<!doctype htm')
parser.split_parse('l public')
parser.split_parse('><html><he')
parser.split_parse('ad><t')
parser.split_parse('itle> Hum... ')
parser.split_parse('A Tit')
parser.split_parse('\nle</ti\ntle>')
parser.split_parse('</\nhead><body>\t\t\t<h3>')
parser.split_parse('Nice Test!</h3>')
parser.split_parse('</bo\n\n\ndy></html>')
parser.finalize()
assert parser.tags.len == 11
assert parser.tags[3].content == ' Hum... A Tit\nle'
}
fn test_giant_string() {
mut temp_html := strings.new_builder(200)
mut parser := Parser{}
temp_html.write_string('<!doctype html><html><head><title>Giant String</title></head><body>')
for counter := 0; counter < 2000; counter++ {
temp_html.write_string("<div id='name_${counter}' class='several-${counter}'>Look at ${counter}</div>")
}
temp_html.write_string('</body></html>')
parser.parse_html(temp_html.str())
assert parser.tags.len == 4009
}
fn test_script_tag() {
mut parser := Parser{}
script_content := '\nvar googletag = googletag || {};\ngoogletag.cmd = googletag.cmd || [];if(3 > 5) {console.log("Quoted \'message\'");}\n'
temp_html := '<html><body><script>${script_content}</script></body></html>'
parser.parse_html(temp_html)
assert parser.tags[2].content.len == script_content.len
}
+203
View File
@@ -0,0 +1,203 @@
module html
import strings
import datatypes
pub enum CloseTagType {
in_name
new_tag
}
// Tag holds the information of an HTML tag.
@[heap]
pub struct Tag {
pub mut:
name string
content string
children []&Tag
attributes map[string]string // attributes will be like map[name]value
last_attribute string
class_set datatypes.Set[string]
parent &Tag = unsafe { nil }
position_in_parent int
closed bool
close_type CloseTagType = .in_name
mut:
text_content string
content_is_inner_html bool
}
fn (mut tag Tag) add_parent(t &Tag, position int) {
tag.position_in_parent = position
tag.parent = t
}
fn (mut tag Tag) add_child(t &Tag) int {
tag.children << t
return tag.children.len
}
// text returns the text contents of the tag.
pub fn (tag &Tag) text() string {
if tag.name == 'text' {
return tag.leading_text()
}
if tag.name.len >= 2 && tag.name[..2] == 'br' {
return '\n'
}
mut text_str := strings.new_builder(200)
text_str.write_string(tag.leading_text())
for child in tag.children {
text_str.write_string(child.text())
}
return text_str.str()
}
fn (tag &Tag) leading_text() string {
if tag.text_content.len > 0 {
return tag.text_content
}
if !tag.content_is_inner_html || tag.children.len == 0 || tag.name == 'text' {
return tag.content
}
return ''
}
pub fn (tag &Tag) str() string {
if tag.name == 'text' {
return tag.leading_text()
}
mut html_str := strings.new_builder(200)
html_str.write_string('<${tag.name}')
for key, value in tag.attributes {
html_str.write_string(' ${key}')
if value != '' {
html_str.write_string('="${value}"')
}
}
html_str.write_string(if tag.closed && tag.close_type == .in_name { '/>' } else { '>' })
html_str.write_string(tag.content)
if !tag.content_is_inner_html && tag.children.len > 0 {
for child in tag.children {
html_str.write_string(child.str())
}
}
if !tag.closed || tag.close_type == .new_tag {
html_str.write_string('</${tag.name}>')
}
return html_str.str()
}
// get_tag retrieves the first found child tag in the tag that has the given tag name.
pub fn (tag &Tag) get_tag(name string) ?&Tag {
for child in tag.children {
if child.name == name {
return child
}
if c := child.get_tag(name) {
return c
}
}
return none
}
// get_tags retrieves all child tags recursively in the tag that have the given tag name.
pub fn (tag &Tag) get_tags(name string) []&Tag {
mut res := []&Tag{}
for child in tag.children {
if child.name == name {
res << child
}
res << child.get_tags(name)
}
return res
}
// get_tag_by_attribute retrieves the first found child tag in the tag that has the given attribute name.
pub fn (tag &Tag) get_tag_by_attribute(name string) ?&Tag {
for child in tag.children {
if child.attributes[name] != '' {
return child
}
if c := child.get_tag_by_attribute(name) {
return c
}
}
return none
}
// get_tags_by_attribute retrieves all child tags recursively in the tag that have the given attribute name.
pub fn (tag &Tag) get_tags_by_attribute(name string) []&Tag {
mut res := []&Tag{}
for child in tag.children {
if child.attributes[name] != '' {
res << child
}
res << child.get_tags_by_attribute(name)
}
return res
}
// get_tag_by_attribute_value retrieves the first found child tag in the tag that has the given attribute name and value.
pub fn (tag &Tag) get_tag_by_attribute_value(name string, value string) ?&Tag {
for child in tag.children {
if child.attributes[name] == value {
return child
}
if c := child.get_tag_by_attribute_value(name, value) {
return c
}
}
return none
}
// get_tags_by_attribute_value retrieves all child tags recursively in the tag that have the given attribute name and value.
pub fn (tag &Tag) get_tags_by_attribute_value(name string, value string) []&Tag {
mut res := []&Tag{}
for child in tag.children {
if child.attributes[name] == value {
res << child
}
res << child.get_tags_by_attribute_value(name, value)
}
return res
}
// get_tag_by_class_name retrieves the first found child tag in the tag that has the given class name(s).
pub fn (tag &Tag) get_tag_by_class_name(names ...string) ?&Tag {
for child in tag.children {
mut matched := true
for name in names {
matched = child.class_set.exists(name)
if !matched {
break
}
}
if matched {
return child
}
if c := child.get_tag_by_class_name(...names) {
return c
}
}
return none
}
// get_tags_by_class_name retrieves all child tags recursively in the tag that have the given class name(s).
pub fn (tag &Tag) get_tags_by_class_name(names ...string) []&Tag {
mut res := []&Tag{}
for child in tag.children {
mut matched := true
for name in names {
matched = child.class_set.exists(name)
if !matched {
break
}
}
if matched {
res << child
}
res << child.get_tags_by_class_name(...names)
}
return res
}
+79
View File
@@ -0,0 +1,79 @@
module html
import strings
const html = '<!doctype html>
<html>
<head></head>
<body>
<div id="1st">
<div class="foo bar"></div>
</div>
<div id="2nd">
<div class="foo">
<a href="https://vlang.io/">V</a>
</div>
<div class="bar"></div>
<a href="https://modules.vlang.io/">vlib</a>
<div class="bar">
<div class="bar">
<p>
<div>modules</div>
</p>
<a href="https://vpm.vlang.io/">vpm</a>
</div>
</div>
</div>
<div id="3rd"></div>
</body>
</html>'
fn test_search_tag_by_type() {
mut dom := parse(html)
tag := dom.get_tags(name: 'body')[0]
assert tag.get_tag('div')?.attributes['id'] == '1st'
assert tag.get_tag_by_attribute('href')?.content == 'V'
// TODO: update after improved parsing to not add trailing white space to attribute values
assert tag.get_tag_by_attribute_value('id', '3rd')?.str() == '<div id="3rd" ></div>'
assert tag.get_tag_by_class_name('foo')?.attributes['class'] == 'foo bar'
}
fn test_search_tags_by_type() {
mut dom := parse(html)
tag := dom.get_tags_by_attribute_value('id', '2nd')[0]
assert tag.get_tags('div').len == 5
assert tag.get_tags_by_attribute('href')[2].content == 'vpm'
assert tag.get_tags_by_attribute_value('class', 'bar').len == 3
assert tag.get_tags_by_class_name('bar').len == 3
}
fn generate_temp_html_with_classes() string {
mut temp_html := strings.new_builder(400)
temp_html.write_string('<!doctype html><html><head><title>Giant String</title></head><body>')
temp_html.write_string("<div class='single'>Single</div>")
for counter := 0; counter < 4; counter++ {
temp_html.write_string("<div id='name_${counter}' ")
temp_html.write_string("class='common'>Common No. ${counter}</div>")
}
temp_html.write_string("<div class='complex-0 complex-1 complex-2'>Complex</div>")
temp_html.write_string("<div class='complex-0 complex-2'>Partial</div>")
temp_html.write_string('</body></html>')
return temp_html.str()
}
fn test_search_by_class() {
mut dom := parse(generate_temp_html_with_classes())
tag := dom.get_tags(name: 'body')[0]
single_class_tags := tag.get_tags_by_class_name('single')
common_class_tags := tag.get_tags_by_class_name('common')
complex_class_tags := tag.get_tags_by_class_name('complex-0', 'complex-1', 'complex-2')
partial_class_tags := tag.get_tags_by_class_name('complex-0', 'complex-2')
shuffled_class_tags := tag.get_tags_by_class_name('complex-2', 'complex-0', 'complex-1')
assert single_class_tags.len == 1
assert common_class_tags.len == 4
assert complex_class_tags.len == 1
assert complex_class_tags[0].attributes['class'] == 'complex-0 complex-1 complex-2'
assert partial_class_tags.len == 2
assert shuffled_class_tags.len == 1
assert shuffled_class_tags[0].attributes['class'] == 'complex-0 complex-1 complex-2'
}