Initial project import
This commit is contained in:
commit
23f2d29e32
11957 files changed
+1754577
No files matched your search
@@ -0,0 +1,20 @@
|
||||
# web_crawler
|
||||
This simple web crawler fetches news from the homepage of HN (https://news.ycombinator.com/).
|
||||
|
||||
# Compile and Run
|
||||
|
||||
Use this to generate an executable, and then launch the web crawler:
|
||||
```bash
|
||||
v web_crawler.v
|
||||
./web_crawler
|
||||
```
|
||||
|
||||
And this to compile and launch the web crawler directly:
|
||||
```bash
|
||||
v run web_crawler.v
|
||||
```
|
||||
|
||||
This project shows how to use http.fetch() to get http.Response,
|
||||
and then html.parse() to parse the returned html.
|
||||
|
||||
It's easy, isn't it?
|
||||
@@ -0,0 +1,21 @@
|
||||
import net.http
|
||||
import net.html
|
||||
|
||||
fn main() {
|
||||
site_url := 'https://news.ycombinator.com'
|
||||
resp := http.fetch(
|
||||
url: site_url
|
||||
user_agent: 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
|
||||
)!
|
||||
mut doc := html.parse(resp.body)
|
||||
tags := doc.get_tags_by_attribute_value('class', 'titleline')
|
||||
for i, tag in tags {
|
||||
el := tag.children[0]
|
||||
mut href := el.attributes['href']!
|
||||
if !href.starts_with('http') {
|
||||
href = '${site_url}/${href}'
|
||||
}
|
||||
title := el.content
|
||||
println('${i + 1:2}. title: ${title:-90s} href: ${href}')
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user