回调最佳实现和控制流模式

export function spider (url, nesting, cb) {
    const filename = urlToFilename(url)
    fs.readFile(filename, 'utf8', (err, fileContent) => {
        if (err) {
            if (err.code !== 'ENOENT') {
                return cb(err)
            }

            // The file doesn't exist, so let's download it
            return download(url, filename, (err, requestContent) => {
                if (err) {
                    return cb(err)
                }
                spiderLinks(url, requestContent, nesting, cb)
            })
        }

        // The file already exists, let's process the links
        spiderLinks(url, fileContent, nesting, cb)
    })
}

function spiderLinks (currentUrl, body, nesting, cb) {
    if (nesting === 0) {
        // Remember Zalgo from chapter 3?
        return process.nextTick(cb)
    }

    const links = getPageLinks(currentUrl, body) // (1)
    if (links.length === 0) {
        return process.nextTick(cb)
    }

    function iterate (index) { // (2)
        if (index === links.length) {
            return cb()
        }

        spider(links[index], nesting - 1, function (err) { // (3)
            if (err) {
                return cb(err)
            }
            iterate(index + 1)
        })
    }

    iterate(0) // (4)
}

import { spider } from './spider.js'

const url = process.argv[2]
const nesting = Number.parseInt(process.argv[3], 10) || 1

spider(url, nesting, err => {
    if (err) {
        console.error(err)
        process.exit(1)
    }

    console.log('Download complete')
})

function iterate (index) {
    if (index === tasks.length) {
        return finish()
    }
    const task = tasks[index]
    task(() => iterate(index + 1))
}

function finish () {
    // iteration completed
}

iterate(0)

iterateSeries(collection, iteratorCallback, finalCallback)

export class TaskQueue {
    constructor (concurrency) {
        this.concurrency = concurrency
        this.running = 0
        this.queue = []
    }

    pushTask (task) {
        this.queue.push(task)
        process.nextTick(this.next.bind(this))
        return this
    }

    next () {
        while (this.running < this.concurrency && this.queue.length) {
            const task = this.queue.shift()
            task(() => {
                this.running--
                process.nextTick(this.next.bind(this))
            })
            this.running++
        }
    }
}

import { EventEmitter } from 'events'

export class TaskQueue extends EventEmitter {
    constructor (concurrency) {
        super()
        // ...
    }
    // ...
}

next () {
    if (this.running === 0 && this.queue.length === 0) { // (1)
        return this.emit('empty')
    }
    while (this.running < this.concurrency && this.queue.length) {
        const task = this.queue.shift()
        task((err) => { // (2)
            if (err) {
                this.emit('error', err)
            }
            this.running--
            process.nextTick(this.next.bind(this))
        })
        this.running++
    }
}

function spiderTask (url, nesting, queue, cb) { // (1)
    const filename = urlToFilename(url)
    fs.readFile(filename, 'utf8', (err, fileContent) => {
        if (err) {
            if (err.code !== 'ENOENT') {
                return cb(err)
            }

            return download(url, filename, (err, requestContent) => {
                if (err) {
                    return cb(err)
                }

                spiderLinks(url, requestContent, nesting, queue) // (2)
                return cb()
            })
        }

        spiderLinks(url, fileContent, nesting, queue) // (3)
        return cb()
    })
}

function spiderLinks (currentUrl, body, nesting, queue) {
    if (nesting === 0) {
        return
    }

    const links = getPageLinks(currentUrl, body)
    if (links.length === 0) {
        return
    }

    links.forEach(link => spider(link, nesting - 1, queue))
}

const spidering = new Set() // (1)
export function spider (url, nesting, queue) {
    if (spidering.has(url)) {
        return
    }

    spidering.add(url)
    queue.pushTask((done) => { // (2)
        spiderTask(url, nesting, queue, done)
    })
}

import { spider } from './spider.js'
import { TaskQueue } from './TaskQueue.js'

const url = process.argv[2] // (1)
const nesting = Number.parseInt(process.argv[3], 10) || 1
const concurrency = Number.parseInt(process.argv[4], 10) || 2

const spiderQueue = new TaskQueue(concurrency) // (2)
spiderQueue.on('error', console.error)
spiderQueue.on('empty', () => console.log('Download complete'))

spider(url, nesting, spiderQueue) // (3)

node spider-cli.js https://loige.co 1 4