node实现简单的数据爬虫

前言

我使用的是墨迹天气的页面，因为这个使用的链接简单 页面结构简单并且大都是文字形式

第一步

打开墨迹天气网址 随便点开一个页面

点击F12或者鼠标右键点击检查 查看页面的信息

分析页面内容

使用文字所在的class和标签来定位

编写代码

配置express环境

引入包

const axios = require(‘axios’);
const cheerio = require(‘cheerio’);

获取html信息

定义url

const weatherURL = https://tianqi.moji.com/weather/china/shanghai/shanghai;

访问相应的网页

try {
let weather = await getWeatherTips(weatherURL);
// let str = $${weather};
// console.log(weather)
res.json(weather);
} catch (error) {
res.status(500).json({ error: ‘Internal Server Error’ });
}
});

将url传入相应的方法中 方法进行处理

function getWeatherTips(url) {
  return new Promise(async (resolve, reject) => {//使用异步请求
    try {
      const response = await axios.get(url);//获取url并访问 得到页面
      const html = response.data || "";//获取页面内容
      const $ = cheerio.load(html);//解析成html
      
      const temp2 = $('.wea_alert  em').text().trim();/
      const temp = $('.wea_weather em').text().trim() + '℃'; //这就是刚刚看到的class和标签名 获取里面的text

      const desc = $('.wea_weather b').text().trim();
      const water = $('.wea_about span').text().trim();
      const win = $('.wea_about em').text().trim();
      const tips = $('.wea_tips em').text().trim();
      const words = `今日天气\n${desc}\n温度：${temp}\n湿度：${water}\n风力：${win}\n${tips}\n空气质量${temp2}`;
      // resolve.json({
      //   success: true
      // });
      const word = {
        temp: temp,
        desc: desc,
        water: water,
        win: win,
        tips: tips,
        area: temp2
      }

      // resolve.json({
      //   words: words
      // });
      resolve(word);//输出返回的内容
      
      // res.json({ success: true, data: 123 });
      // resolve.json({ success: true, data: `${words}` });
      // res.json({
      //   success: true, data: {
      //     words:words
      // temp: temp,
      // desc: desc,
      // water: water,
      // win: win,
      // tips: tips,

      // }
      // });

    } catch (error) {
      reject(error);
    }
  });
}

全部代码

这个是我加了传入参数的版本 你可以自己把你定义的天气相应的url设置好

const axios = require('axios');
const cheerio = require('cheerio');

app.post('/words', async function (req, res) {
  const arr = req.body.address;
  console.log(arr)

  const weatherURL = `https://tianqi.moji.com/weather/china` + `${arr}`;
  console.log(weatherURL)
  try {
    let weather = await getWeatherTips(weatherURL);
    // let str = `$${weather}`;
    // console.log(weather)
    res.json(weather);
  } catch (error) {
    res.status(500).json({ error: 'Internal Server Error' });
  }
});

// 获取墨迹天气提示信息
function getWeatherTips(url) {
  return new Promise(async (resolve, reject) => {
    try {
      const response = await axios.get(url);
      const html = response.data || "";
      const $ = cheerio.load(html);
      const temp2 = $('.wea_alert  em').text().trim();
      const temp = $('.wea_weather em').text().trim() + '℃';

      const desc = $('.wea_weather b').text().trim();
      const water = $('.wea_about span').text().trim();
      const win = $('.wea_about em').text().trim();
      const tips = $('.wea_tips em').text().trim();
      const words = `今日天气\n${desc}\n温度：${temp}\n湿度：${water}\n风力：${win}\n${tips}\n空气质量${temp2}`;
      // resolve.json({
      //   success: true
      // });
      const word = {
        temp: temp,
        desc: desc,
        water: water,
        win: win,
        tips: tips,
        area: temp2
      }

      // resolve.json({
      //   words: words
      // });
      resolve(word);
      // res.json({ success: true, data: 123 });
      // resolve.json({ success: true, data: `${words}` });
      // res.json({
      //   success: true, data: {
      //     words:words
      // temp: temp,
      // desc: desc,
      // water: water,
      // win: win,
      // tips: tips,

      // }
      // });

    } catch (error) {
      reject(error);
    }
  });
}

接口测试