利用Python多线程实现图片下载器

search_urls, pagesize = [], 30
for pn in range(math.ceil(search_limits * 1.2 / pagesize)):
    search_url = base_url.fORMat(quote(keyword), quote(keyword), pn * pagesize, pagesize)
    search_urls.append(search_url)

# 多线程请求获取所有图片链接
def searchapi(self, search_urls, image_urls, bar):
    while len(search_urls) > 0:
        search_url = search_urls.pop(0)
        response = self.get(search_url)
        if response is None: 
            bar()
            continue
        response.encoding = 'utf-8'
        response_json = json.loads(response.text.replace(r"\'", ""), encoding='utf-8', strict=False)
        for item in response_json['data']:
            if 'objURL' in item.keys():
                image_urls.add(self.parseurl(item['objURL']))
            elif 'replaceUrl' in item.keys() and len(item['replaceUrl']) == 2:
                image_urls.add(item['replaceUrl'][1]['ObjURL'])
        bar()
task_pool, image_urls = [], set()
with alive_bar(min(len(search_urls), search_limits)) as bar:
    for idx in range(num_threadings):
        task = threading.Thread(
            target=searchapi,
            args=(self, search_urls, image_urls, bar)
        )
        task_pool.append(task)
        task.start()
    for task in task_pool: task.join()

'''下载'''
def download(self, keyword, search_limits=1000, num_threadings=5, savedir='outputs'):
    touchdir(savedir)
    # 获得image_urls
    self.logging(f'Start to search images from {self.source_name}')
    image_urls = self.search(keyword, search_limits, num_threadings)
    # 多线程下载图片
    self.logging(f'Start to download images from {self.source_name}')
    def downloadapi(self, savepaths, image_urls, bar):
        assert len(savepaths) == len(image_urls)
        while len(image_urls) > 0:
            savepath, image_url = savepaths.pop(0), image_urls.pop(0)
            response = self.get(image_url)
            if response is None: 
                bar()
                continue
            with open(savepath, 'wb') as fp: fp.write(response.content)
            filetype = imghdr.what(savepath)
            if filetype in ['jpg', 'jpeg', 'png', 'bmp', 'gif']:
                savepath_correct = f'{savepath}.{filetype}'
                shutil.move(savepath, savepath_correct)
            else:
                os.remove(savepath)
            bar()
    task_pool, savepaths = [], []
    for idx in range(len(image_urls)):
        savename = f'image_{str(idx).zfill(8)}'
        savepaths.append(os.path.join(savedir, savename))
    with alive_bar(len(image_urls)) as bar:
        for idx in range(num_threadings):
            task = threading.Thread(
                target=downloadapi,
                args=(self, savepaths, image_urls, bar)
            )
            task_pool.append(task)
            task.start()
        for task in task_pool: task.join()

# 构建所有urls
base_url = 'Https://cn.bing.com/images/async?q={}&first={}&count={}&cw=1536&ch=240&relp={}&tsc=ImageBasicHover&datsrc=I&layout=RowBased&mmasync=1&dgState=x*1063_y*768_h*186_c*5_i*71_r*10&IG=D6A4AD486F3A49F1BE164BC50750D641&SFX=3&iid=images.5555'
search_urls, pagesize = [], 35
for pn in range(math.ceil(search_limits * 1.2 / pagesize)):
    search_url = base_url.format(quote(keyword), pn * pagesize, pagesize, pagesize)
    search_urls.append(search_url)

# 构建所有urls
base_url = 'https://www.Google.com/search?'
search_urls, pagesize = [], 20
for pn in range(math.ceil(search_limits * 1.2 / pagesize)):
    params = {
        'q': keyword,
        'ijn': pn,
        'start': pn * pagesize,
        'tbs': '',
        'tbm': 'isch',
    }
    search_urls.append(base_url + urlencode(params))

pip install pyimagedl

Usage: imagedl [OPTioNS]

Options:
  --version                  Show the version and exit.
  -k, --keyword TEXT         想要搜索下载的图片关键字, 若不指定, 则进入imagedl终端版
  -s, --savedir TEXT         下载的图片的保存路径
  -t, --target TEXT          指定图片搜索下载的平台, 例如"baidu"
  -l, --limits INTEGER       下载的图片数量
  -n, --nthreadings INTEGER  使用的线程数量
  --help                     Show this message and exit.

imagedl -k 狗狗 -s dogs -t baidu -l 1000