node实现简单的数据爬虫

前言

我使用的是墨迹天气的页面，因为这个使用的链接简单 页面结构简单并且大都是文字形式

第一步

打开墨迹天气网址 随便点开一个页面 点击F12或者鼠标右键点击检查 查看页面的信息

分析页面内容

使用文字所在的class和标签来定位

编写代码 配置express环境 引入包

const axios = require(‘axios’); const cheerio = require(‘cheerio’);

获取html信息 定义url

const weatherURL = tianqi.moji /weather/china/shanghai/shanghai;

访问相应的网页

try { let weather = await getWeatherTips(weatherURL); // let str = $${weather}; // console.log(weather) res.json(weather); } catch (error) { res.status(500).json({ error: ‘Internal Server Error’ }); } });

将url传入相应的方法中 方法进行处理 function getWeatherTips(url) { return new Promise(async (resolve, reject) => {//使用异步请求 try { const response = await axios.get(url);//获取url并访问 得到页面 const html = response.data || "";//获取页面内容 const $ = cheerio.load(html);//解析成html const temp2 = $('.wea_alert em').text().trim();/ const temp = $('.wea_weather em').text().trim() + '℃'; //这就是刚刚看到的class和标签名 获取里面的text const desc = $('.wea_weather b').text().trim(); const water = $('.wea_about span').text().trim(); const win = $('.wea_about em').text().trim(); const tips = $('.wea_tips em').text().trim(); const words = `今日天气\n${desc}\n温度：${temp}\n湿度：${water}\n风力：${win}\n${tips}\n空气质量${temp2}`; // resolve.json({ // success: true // }); const word = { temp: temp, desc: desc, water: water, win: win, tips: tips, area: temp2 } // resolve.json({ // words: words // }); resolve(word);//输出返回的内容 // res.json({ success: true, data: 123 }); // resolve.json({ success: true, data: `${words}` }); // res.json({ // success: true, data: { // words:words // temp: temp, // desc: desc, // water: water, // win: win, // tips: tips, // } // }); } catch (error) { reject(error); } }); } 全部代码

这个是我加了传入参数的版本 你可以自己把你定义的天气相应的url设置好

const axios = require('axios'); const cheerio = require('cheerio'); app.post('/words', async function (req, res) { const arr = req.body.address; console.log(arr) const weatherURL = ` tianqi.moji /weather/china` + `${arr}`; console.log(weatherURL) try { let weather = await getWeatherTips(weatherURL); // let str = `$${weather}`; // console.log(weather) res.json(weather); } catch (error) { res.status(500).json({ error: 'Internal Server Error' }); } }); // 获取墨迹天气提示信息 function getWeatherTips(url) { return new Promise(async (resolve, reject) => { try { const response = await axios.get(url); const html = response.data || ""; const $ = cheerio.load(html); const temp2 = $('.wea_alert em').text().trim(); const temp = $('.wea_weather em').text().trim() + '℃'; const desc = $('.wea_weather b').text().trim(); const water = $('.wea_about span').text().trim(); const win = $('.wea_about em').text().trim(); const tips = $('.wea_tips em').text().trim(); const words = `今日天气\n${desc}\n温度：${temp}\n湿度：${water}\n风力：${win}\n${tips}\n空气质量${temp2}`; // resolve.json({ // success: true // }); const word = { temp: temp, desc: desc, water: water, win: win, tips: tips, area: temp2 } // resolve.json({ // words: words // }); resolve(word); // res.json({ success: true, data: 123 }); // resolve.json({ success: true, data: `${words}` }); // res.json({ // success: true, data: { // words:words // temp: temp, // desc: desc, // water: water, // win: win, // tips: tips, // } // }); } catch (error) { reject(error); } }); } 接口测试