煎蛋模块化爬虫

按照小甲鱼的Python课程打出来的,在find_imgs中添加了http:(未使用正则表达式)

煎蛋模块化爬虫

import os
import urllib.request

def url_open(url):
    req = urllib.request.Request(url)
    req.add_header('User-Agent', '')
    response = urllib.request.urlopen(url)
    html = response.read()

    return html



def get_page(url):
    html = url_open(url).decode('utf-8')

    a = html.find('current-comment-page') + 23
    b = html.find(']',a)

    return html[a:b]



def find_imgs(url):
    html = url_open(url).decode('utf-8')
    img_addrs = []

    a = html.find('img src=')

    while a != -1:
        b = html.find('.jpg', a, a+255)
        if b != -1:
            img_addrs.append('http:'+html[a+9:b+4])
        else:
            b =a + 9

        a = html.find('img src=', b)

    return img_addrs



def save_imgs(folder, img_addrs):
    for each in img_addrs:
        filename = each.split('/')[-1]
        with open(filename, 'wb') as f:
            img = url_open(each)
            f.write(img)



def download_mm(folder= 'OOXX', pages=100):
    os.mkdir(folder)
    os.chdir(folder)


    url = 'http://jandan.net/ooxx/'
    page_num = int(get_page(url))


    for i in range(pages):
        page_num -=i
        page_url = url + 'page-' +str(page_num)
        img_addrs = find_imgs(page_url)
        save_imgs(folder, img_addrs)

download_mm()#//因为有默认参数所以()可以为空,没有默认参数时不可以

find()用法:

#!/usr/bin/python

str1 = "this is string example....wow!!!"
str2 = "exam"

print (str1.find(str2))
print (str1.find(str2, 10))
print (str1.find(str2, 10,12))
print (str1.find(str2, 40))

输出:
15
15
-1
-1