<1> 线程池—ThreadPoolExecutor

 1 import time
 2 from concurrent.futures import ThreadPoolExecutor, as_completed, ProcessPoolExecutor
 3 
 4 
 5 class Spider:
 6     
 7     executor = ThreadPoolExecutor(max_workers=10)
 8     # executor = ProcessPoolExecutor(max_workers=10)
 9     
10     def __init__(self):
11         pass
12 
13     def request(self, url, second): # 模拟耗时任务
14         time.sleep(second)
15         return url, second
16 
17     def run(self):
18         all_tasks = []
19         for index in range(100):  # index ----------> 模拟url
20             task = self.executor.submit(self.http_request, index, 2)
21             all_tasks.append(task)
22 
23         for result in as_completed(all_tasks):
24             exception = result.exception()
25             if exception:
26                 pass
27             data = result.result()
28             print(data)
29 
30 
31 if __name__ == '__main__':
32     s = Spider()
33     s.run()