Python スクレイピングで使えそうな6つのコード片

import requests


def download_mp3(url, fname):
    doc = requests.get(url)
    with open(fname, 'wb') as f:
        f.write(doc.content)

import ffmpeg


def download_mp4(url, fname):
    stream = ffmpeg.input(url)

    # 画像フィルタによって音声トラックが消えてしまう場合があるので,
    # 先に音声を分けておく
    audio_stream = stream.audio

    # 画像を反転する
    stream = ffmpeg.hflip(stream)

    stream = ffmpeg.output(stream, audio_stream, './data/' + fname + '.mp4')
    ffmpeg.run(stream)

from selenium import webdriver

def get_html(url):
    driver = webdriver.Chrome()
    driver.get(url)
    html = driver.page_source
    driver.quit()
    return html

def main():
   html = get_html(url)
   res = BeautifulSoup(res, 'html.parser')
   # ...

import requests
url_redirected = requests.get(url).url

import requests

cookie = {'has_js': '1'}
headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0"}
response = requests.get(url=url, headers=headers, cookies=cookie)
html = response.content
BeautifulSoup(html, "html.parser")

# async.py
import asyncio


async def sleep(c):
    print("start", c)
    await asyncio.sleep(c)
    print("end", c)


async def doit():
    await asyncio.gather(*[sleep(c) for c in [6, 5, 4, 3, 2, 1]])


def main():
    loop = asyncio.get_event_loop()
    loop.run_until_complete(doit())
    loop.close()


if __name__ == '__main__':
    main()

$ python async.py
loop = asyncio.get_event_loop()
start 6
start 5
start 4
start 3
start 2
start 1
end 1
end 2
end 3
end 4
end 5
end 6