下载漫画只需一个脚本

近来比较闲，想利用这闲暇的时光恶补下op漫画。打开鼠绘漫画网，找到onepiece的目录，点击需要补习的篇章。本来准备吹着空调，吃着西瓜愉快的享受op的世界了。没想到的事发生了，在鼠标点击的瞬间出现了跳转

然后就眼睁睁的跳转到腾讯蛋疼的简体汉化版，此时心中无数个草泥马在奔腾啊！！！

由于版权纷争，鼠绘为了生计不得不退出。并且转型为漫画讨论社区，虽然最近几期都给了第三方链接，但是总给人的感觉是偷呢？好在，手机端APP还保留着漫画阅读功能，但是由于屏幕狭小，体验不是很好。于是就有了本篇简单爬虫想法。

前期利用App进行抓包调研鼠绘网后台接口，得到了几个关键的接口。之后就写了如下脚本:
脚本下载地址：cartoon_download.py

#! /usr/bin/python
# -*- coding:utf-8 -*-
import urllib
import socket
import json
import os
import HTMLParser
import threading
import time
import sys


global jsonstr, hostname, croot, maxtimeout, tryagaincount
global threadcount, currentthreadnum, mutex, thenewest

hostname = r'http://www.ishuhui.net'
chapters = r'/ComicBooks/GetChapterList?id=' + '%d' + '&PageIndex=' + '%d'
details = r'/ComicBooks/ReadComicBooksToIsoV1/' + '%d'
repo = {
    'op': 2,
    'SLAM_DUNK': 38,
    '火影忍者': 4,
    '银魂': 10,
    '妖精的尾巴': 3,
    '名侦探柯南': 1,
    'bleach': 23,
    '黑子的篮球': 6,
    '浪客剑心': 39,
    '结界师': 34
}
arg = '[op, SLAM_DUNK, 火影忍者, 银魂, 妖精的尾巴, 名侦探柯南, bleach, 黑子的篮球, 浪客剑心, 结界师]'
cartoonid = repo['op']
thenewest = 0
currentthreadnum = 0
threadcount = 6
tryagaincount = 5
maxtimeout = 30
croot = os.getcwd()
mutex = threading.Lock()
usage = \
    """
    Usage:
            cartoon_download [args...]

            cartoon_download cartoon
            cartoon_download cartoon path
            cartoon_download cartoon path newestcount
            cartoon_download cartoon path newestcount threadcount

    For example:
            cartoon_download op /home/xxx/onepiece

    Note:
            current version just support Linux/Unix os.
            cartoon = %s
            path can be either absolute or relative, but must be en characters.
            default `newestcount` value is 0 to download all chapters, or
            download the newest value chapters.
    """ % arg


if len(sys.argv) == 1:
    print usage
    sys.exit(0)
if len(sys.argv) >= 2:
    try:
        cartoonid = repo[sys.argv[1]]
    except Exception, e:
        print 'Please select from %s' % arg
        sys.exit(0)
    finally:
        pass
if len(sys.argv) >= 3:
    targetdir = sys.argv[2]
    if not os.path.exists(targetdir):
        os.makedirs(targetdir)
    croot = os.path.abspath(targetdir)
    if len(sys.argv) >= 4:
        thenewest = (int)(sys.argv[3])
    if len(sys.argv) == 5:
        threadcount = (int)(sys.argv[4])


class MyHTMLParser(HTMLParser.HTMLParser):
    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.gifs = []
        self.jpgs = []
        self.pngs = []

    def handle_starttag(self, tags, attrs):
        if tags == 'img':
            for attr in attrs:
                for htmlstr in attr:
                    if 'gif' in htmlstr:
                        self.gifs.append(htmlstr)
                    elif 'jpg' in htmlstr:
                        self.jpgs.append(htmlstr)
                    elif 'png' in htmlstr:
                        self.pngs.append(htmlstr)
                    else:
                        pass

    def get_gifs(self):
        return self.gifs

    def get_jpgs(self):
        return self.jpgs

    def get_pngs(self):
        return self.pngs


class DownloadTask(threading.Thread):

    def __init__(self, name, srcurl):
        threading.Thread.__init__(self)
        self.name = name
        self.srcurl = srcurl

    def run(self):
        pic2file(self.srcurl)


def createtask(imgs):
    global mutex, currentthreadnum, threadcount
    print 'current tasks num >> %d' % len(imgs)
    for srcurl in imgs:
        while (currentthreadnum >= threadcount):
            time.sleep(0.5)
        increasethread()
        threadname = '#%s' % srcurl
        task = DownloadTask(threadname, srcurl)
        task.start()
    while (currentthreadnum > 0):
        time.sleep(0.5)
    print 'finished!'
    os.chdir(croot)


def parser2name(picurl):
    names = picurl.split('/')
    result = names[len(names) - 1]
    if '?' in result:
        result = result.split('?')[0]
    return result


def increasethread():
    global currentthreadnum, mutex
    mutex.acquire()
    currentthreadnum += 1
    mutex.release()


def decreasethread():
    global currentthreadnum, mutex
    mutex.acquire()
    currentthreadnum -= 1
    mutex.release()


def pic2file(picurl, times=0):
    protocol = picurl.split('/')[0]
    if not(protocol == 'http:' or protocol == 'https:'):
        decreasethread()
        return
    filename = parser2name(picurl)
    if(os.path.exists(filename)):
        decreasethread()
        return
    try:
        pic = urllib.urlopen(picurl)
        data = pic.read()
        picfile = open('%s' % filename, 'wb')
        picfile.write(data)
        picfile.close()
        decreasethread()
    except socket.timeout:
        if times < tryagaincount:
            print "Download '%s' timeout, Trying again." % filename
            pic2file(picurl, times + 1)
        else:
            decreasethread()
            print "Tried %d times, but still failed to %s." %\
                (tryagaincount, filename)
    except Exception as e:
        print('---pic2file error---', e)
        if times < tryagaincount:
            print "Download '%s' timeout, Trying again." % filename
            pic2file(picurl, times + 1)
        else:
            decreasethread()
            print "Task '%s' failed after tring %d times" %\
                (picurl, tryagaincount)
    finally:
        pass


def fixurl(picurl):
    piece = picurl.split('/')
    url = 'http:'
    for p in xrange(1, len(piece)):
        url += '/%s' % piece[p]
    return url


def fetchres(detailurl, dirpath, times=0):
    try:
        if not os.path.exists(dirpath):
            os.mkdir(dirpath)
        os.chdir(dirpath)
        curdir = os.getcwd()
        print 'Download for ' + curdir
        detailbook = urllib.urlopen(detailurl).read()
        htmlfile = open('%s.html' % parser2name(curdir), 'wb')
        htmlfile.write(detailbook)
        htmlfile.close()
        parser = MyHTMLParser()
        parser.feed(detailbook)
        jpgs = parser.get_jpgs()
        pngs = parser.get_pngs()
        gifs = parser.get_gifs()
        imgs = jpgs + pngs + gifs
        createtask(imgs)
    except socket.timeout:
        print "Fetch '%s' timeout." % detailurl
        if times < tryagaincount:
            print "The no.%d times to try." % (times + 2)
            fetchres(detailurl, dirpath, times + 1)
        else:
            print "Tried %d times, but still failed."
            print "####Please check network!####"
    except Exception, e:
        print(e)
    finally:
        pass


def jsonparse():
    encodejson = json.loads(jsonstr)
    result = encodejson["Return"]["List"]
    if thenewest > 0:
        for i in xrange(0, thenewest):
            parserandsavehtml(result[i])
        return False
    else:
        for x in result:
            parserandsavehtml(x)
        return len(result)


def parserandsavehtml(item):
    bookid = item["Id"]
    detailurl = hostname + details % bookid + '.html'
    chapterno = item["ChapterNo"]
    title = item["Title"]
    dirname = '%d %s' % (chapterno, title)
    # Here cause an error `UnicodeEncodeError` when
    # the 'croot' include chinese characters.
    # details info can seach for http://www.cnblogs.com/abcat/p/3389531.html
    targetpath = os.path.join(croot, dirname)
    fetchres(detailurl, targetpath)


def calculatetime(used):
    if used <= 60:
        print 'Total used time is %ds.' % used
    elif used <= 3600:
        print 'Total used time is %dmins %ds.' % (used / 60, used % 60)
    else:
        print 'Total used time is %dhrs %dmins %ds.' %\
            (used / 3600, (used % 3600) / 60, (used % 3600) % 60)


if __name__ == '__main__':
    os.chdir(croot)
    socket.setdefaulttimeout(maxtimeout)
    start = time.time()
    try:
        i = 0
        isbreak = True
        while isbreak:
            targetweb = hostname + chapters % (cartoonid, i)
            webfile = urllib.urlopen(targetweb)
            jsonstr = webfile.read()
            isbreak = jsonparse()
            i = i + 1
        end = time.time()
        calculatetime(int(end - start))
    except socket.timeout:
        print('timeout')
    except Exception as e:
        print('error', e)
    finally:
        pass

最后在po一张路飞仙人模式图O(∩_∩)O

声明：本脚本仅供学习交流使用，禁止用于商业用途，若带来商业纠纷与本人无关！