import threading
import requests
max_concurrent = 5
semaphore = threading.Semaphore(max_concurrent)
def fetch(url):
with semaphore:
response = requests.get(url, timeout=5)
return response.text
urls = ["https://example.com"] * 10
threads = [threading.Thread(target=fetch, args=(u,)) for u in urls]
for t in threads:
t.start()
for t in threads:
t.join()
import asyncio
import aiohttp
MAX_CONCURRENT = 10
semaphore = asyncio.Semaphore(MAX_CONCURRENT)
async def fetch(session, url):
async with semaphore:
async with session.get(url) as resp:
return await resp.text()
async def main():
urls = ["https://example.com"] * 20
async with aiohttp.ClientSession() as session:
tasks = [fetch(session, u) for u in urls]
results = await asyncio.gather(*tasks)
print(len(results))
asyncio.run(main())
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
MAX_WORKERS = 4
def fetch(url):
resp = requests.get(url, timeout=5)
return resp.text
urls = ["https://example.com"] * 8
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
future_to_url = {executor.submit(fetch, u): u for u in urls}
for future in as_completed(future_to_url):
data = future.result()
# process data