asyncc实践

Posted on | In | | Visitors: ℃
Words count in article: 430 | Reading time ≈ 2

协程异步实践

aiohttp: 是一个为Python提供异步HTTP 客户端/服务端编程，基于asyncio(Python用于支持异步编程的标准库)的异步库。

​ 此代码，通过aiohttp异步请求四个网页，并且异步解析网页，存储为四个文件。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
"""
It's example of usage asyncio+aiohttp to downloading.
You should install aiohttp for using:
(You can use virtualenv to testing)
pip install -r /path/to/requirements.txt
"""

import asyncio
import concurrent.futures as cofu
from os.path import basename

import aiohttp


def download(ways):
    if not ways:
        print('Ways list is empty. Downloading is impossible')
        return

    print('downloading..')

    success_files = set()
    failure_files = set()

    event_loop = asyncio.get_event_loop()
    try:
        event_loop.run_until_complete(
            async_downloader(ways, event_loop, success_files, failure_files)
        )
    finally:
        event_loop.close()

    print('Download complete')
    print('-' * 100)

    if success_files:
        print('success:')
        for file in success_files:
            print(file)

    if failure_files:
        print('failure:')
        for file in failure_files:
            print(file)

#请求网页
async def async_downloader(ways, loop, success_files=set(), failure_files=set()):
    async with aiohttp.ClientSession() as session:
        coroutines = [
            download_file_by_url(
                url,
                session=session,
            ) for url in ways]

        completed, pending = await asyncio.wait(coroutines, return_when=cofu.FIRST_COMPLETED)
        while pending:

            for task in completed:
                fail, url = task.result()

                if fail:
                    failure_files.add(url)
                else:
                    success_files.add(url)

            completed, pending = await asyncio.wait(pending, return_when=cofu.FIRST_COMPLETED)

        for task in completed:
            fail, url = task.result()
            if fail:
                failure_files.add(url)
            else:
                success_files.add(url)

#解析网页
async def download_file_by_url(url, session=None):
    fail = True
    file_name = basename(url)
    assert session

    try:
        async with session.get(url) as response:
            if response.status == 404:
                print('\t{} from {} : Failed : {}'.format(file_name, url, '404 - Not found'))
                return fail, url

            if not response.status == 200:
                print('\t{} from {} : Failed : HTTP response {}'.format(file_name, url, response.status))
                return fail, url

            data = await response.read()

            with open(file_name, 'wb') as file:
                file.write(data)

    except asyncio.TimeoutError as err:
        print('\t{} from {}: Failed : {}'.format(file_name, url, 'Timeout error'))

    except aiohttp.client_exceptions.ClientConnectionError as err:
        print('\t{} from {}: Failed : {}'.format(file_name, url, 'Client connection error'))

    else:
        print('\t{} from {} : Success'.format(file_name, url))
        fail = False

    return fail, url

#多多益善
def test():
    ways = ['https://www.baidu.com',
            'https://www.jd.com',
            'https://www.cn.bing.com',
            'https://www.snailfrying.github.io',
            ]

    download(ways)


if __name__ == "__main__":
    test()