Rust 爬虫(简单)


代码来自于 【原子之音】 视频,感谢up主.

主要测试了三个模块: tokio surf reqwest
surf【使用文档】
tokio【使用文档】 或者查看 tokio中文文档
reqwest【使用文档】

方法一、tokio + reqwest + futures

[dependencies]
reqwest = {version="0.10.10",feature=["json"]}
tokio = {version="0.2",feature=["full"]}
futures = "0.3"
use futures::future::join_all;


async fn fetch_path(path:String) -> Result<String,reqwest::Error>{
    let mut back_string = String::new();
    match reqwest::get(&path).await {
        Ok(response) => {
            match response.text().await{
                Ok(text) =>{
                    println!("Read response text {},{}" ,text.len(),text);
                    back_string = format!("Read response text {} \t {}\t {}",path,text.len(),text)
                }
                Err(_) => {
                    println!("Read response text Error!")
                }
            };
        }
        Err(_) => {
            println!("reqwest get Error!")
        }
    }
    Ok(back_string)
}


#[tokio::main]
async fn main() -> Result<(),reqwest::Error>{
    let paths = vec![
        "https://blog.csdn.net/xxxx".to_string(),
        "https://blog.csdn.net/x/article/details/xxxx".to_string()
    ];
    let result_list = join_all(paths.into_iter().map(|path|{
        fetch_path(path)
    })).await;

    let mut list_string:Vec<String> = vec![];
    for ele in result_list.into_iter(){
        if ele.is_ok(){
            list_string.push(ele.unwrap())
        }else {
            return Err(ele.unwrap_err())
        }
    }

    println!("请求输出:{:?}",list_string);
    Ok(())
}

不知为何,编译报错:

error[E0433]: failed to resolve: could not find `main` in `tokio`
  --> src\main.rs:26:10
   |
26 | #[tokio::main]
   |          ^^^^ could not find `main` in `tokio`

方法二、async-std + surf + futures

[dependencies]
futures = "0.3"
surf = "2.1.0"


[dependencies.async-std]
version = "1.7.0"
features = ["attributes"]
use futures::future::join_all;


async fn fetch_path(path:String) -> surf::Result<String>{
    let mut back_string = String::new();
    match surf::get(&path).await {
        Ok(mut response) => {
            match response.body_string().await{
                Ok(text) =>{
                    println!("Read response text {},{}" ,text.len(),text);
                    back_string = format!("Read response text {} \t {}\t {}",path,text.len(),text)
                }
                Err(_) => {
                    println!("Read response text Error!")
                }
            };
        }
        Err(_) => {
            println!("reqwest get Error!")
        }
    }
    Ok(back_string)
}


#[async_std::main]
async fn main() -> surf::Result<()>{
    let paths = vec![
        "https://blog.csdn.net/xxxx".to_string(),
        "https://blog.csdn.net/xxxxx/xxx".to_string()
    ];
    let result_list = join_all(paths.into_iter().map(|path|{
        fetch_path(path)
    })).await;

    let mut list_string:Vec<String> = vec![];
    for ele in result_list.into_iter(){
        if ele.is_ok(){
            list_string.push(ele.unwrap())
        }else {
            return Err(ele.unwrap_err())
        }
    }

    println!("请求输出:{:?}",list_string);
    Ok(())
}

运行正常

已标记关键词 清除标记
相关推荐
©️2020 CSDN 皮肤主题: 1024 设计师:白松林 返回首页