Nodejs实现定时爬虫的完整实例

const axios = require('axios')
const roomid = "146088"
const ruid = "642922"
const url = `https://api.live.bilibili.com/xlive/app-room/v2/guardTab/topList?roomid=${roomid}&ruid=${ruid}&page_size=30`

const Captin = {
 1: '总督',
 2: '提督',
 3: '舰长'
}

const reqPromise = url => axios.get(url);

let CaptinList = []
let UserList = []

async function crawler(URL, pageNow) {
 const res = await reqPromise(URL);
 if (pageNow == 1) {
 CaptinList = CaptinList.concat(res.data.data.top3);
 }
 CaptinList = CaptinList.concat(res.data.data.list);
}


function getMaxPage(res) {

 const Info = res.data.data.info
 const { page: maxPage } = Info
 return maxPage
}


function getUserList(res) {

 for (let item of res) {
 const userInfo = item
 const { uid, username, guard_level } = userInfo
 UserList.push({ uid, username, Captin: Captin[guard_level] })
 }
}

async function main(UID) {
 const maxPage = await reqPromise(`${url}&page=1`).then(getMaxPage)
 for (let pageNow = 1; pageNow < maxPage + 1; pageNow++) {
 const URL = `${url}&page=${pageNow}`;
 await crawler(URL, pageNow);
 }
 getUserList(CaptinList)
 console.log(search(UID, UserList))
 return search(UID, UserList)
}

function search(uid, UserList) {
 for (let i = 0; i < UserList.length; i++) {
 if (UserList[i].uid === uid) {
 return UserList[i];
 }
 }
 return 0
}

module.exports = {
 main
}

const Koa = require('koa');
const app = new Koa();
const path = require('path')
const fs = require('fs');
const router = require('koa-router')();
const index = require('./index')
const views = require('koa-views')



app.use(views(path.join(__dirname, './'), {
 extension: 'ejs'
}))
app.use(router.routes());

router.get('/', async ctx => {
 ctx.response.type = 'html';
 ctx.response.body = fs.createReadStream('./index.html');
})

router.get('/api/captin', async (ctx) => {
 const UID = ctx.request.query.uid
 console.log(UID)
 const Info = await index.main(parseInt(UID))
 await ctx.render('index', {
 Info,
 })
});

app.listen(3000);

function throttle(fn, delay) {
 var timer;
 return function () {
 var _this = this;
 var args = arguments;
 if (timer) {
  return;
 }
 timer = setTimeout(function () {
  fn.apply(_this, args);
  timer = null; // 在delay后执行完fn之后清空timer，此时timer为假，throttle触发可以进入计时器
 }, delay)
 }
}

npm install node-schedule
# 或
yarn add node-schedule

const schedule = require('node-schedule');

const job = schedule.scheduleJob('42 * * * *', function(){
 console.log('The answer to life, the universe, and everything!');
});

const schedule = require('node-schedule');

// 定义一个时间
let date = new Date(2021, 3, 10, 12, 00, 0);

// 定义一个任务
let job = schedule.scheduleJob(date, () => {
 console.log("现在时间:",new Date());
});

const schedule = require('node-schedule');

// 定义规则
let rule = new schedule.RecurrenceRule();
rule.second = 0
//每分钟 0 秒执行一次

// 启动任务
let job = schedule.scheduleJob(rule, () => {
 console.log(new Date());
});

job.cancel();