>>> def simple_coroutine():
...  print('coroutine start')
...  x = yield
...  print('coroutine recive:',x)
...  
>>> my_co=simple_coroutine()
>>> my_co
<generator object simple_coroutine at 0x1085174f8>
>>> next(my_co)
coroutine start
>>> my_co.send(42)
coroutine recive: 42
Traceback (most recent call last):
 File "<input>", line 1, in <module>
StopIteration

>>> import inspect
>>> inspect.getgeneratorstate(my_co)
'GEN_CLOSED'

>>> my_co=simple_coroutine()
>>> my_co.send(42)
Traceback (most recent call last):
 File "<input>", line 1, in <module>
TypeError: can't send non-None value to a just-started generator
>>> my_co=simple_coroutine()
>>> my_co.send(None)
coroutine start

>>> def simple_coro2(a):
...  print('-> Started: a =', a)
...  b = yield a
...  print('-> Received: b =', b)
...  c = yield a + b
...  print('-> Received: c =', c)
...  
>>> my_coro2 = simple_coro2(14)
>>> from inspect import getgeneratorstate
>>> getgeneratorstate(my_coro2)
'GEN_CREATED'
>>> next(my_coro2) # 協(xié)程執(zhí)行到`b = yield a`處暫停，等待為b賦值，
-> Started: a = 14
14
>>> getgeneratorstate(my_coro2) 
'GEN_SUSPENDED' #從狀態(tài)也可以看到，當(dāng)前是暫停狀態(tài)。
>>> my_coro2.send(28) #將28發(fā)送到協(xié)程，計算yield表達(dá)式，并把結(jié)果綁定到b，產(chǎn)出a+b的值，然后暫停。
-> Received: b = 28
42
>>> my_coro2.send(99)
-> Received: c = 99
Traceback (most recent call last):
 File "<input>", line 1, in <module>
StopIteration
>>> getgeneratorstate(my_coro2)
'GEN_CLOSED'

from bs4 import BeautifulSoup
import requests
from urllib.parse import urlparse

start_url = 'https://www.cnblogs.com'
trust_host = 'www.cnblogs.com'
ignore_path = []
history_urls = []


def parse_html(html):
  soup = BeautifulSoup(html, "lxml")
  print(soup.title)
  links = soup.find_all('a', href=True)
  return (a['href'] for a in links if a['href'])


def parse_url(url):
  url = url.strip()

  if url.find('#') >= 0:
    url = url.split('#')[0]
  if not url:
    return None
  if url.find('javascript:') >= 0:
    return None

  for f in ignore_path:
    if f in url:
      return None
  if url.find('http') < 0:
    url = start_url + url
    return url
  parse = urlparse(url)
  if parse.hostname == trust_host:
    return url


def consumer():
  html = ''
  while True:
    url = yield html
    if url:
      print('[CONSUMER] Consuming %s...' % url)
      rsp = requests.get(url)
      html = rsp.content


def produce(c):
  next(c)

  def do_work(urls):
    for u in urls:
      if u not in history_urls:
        history_urls.append(u)
        print('[PRODUCER] Producing %s...' % u)
        html = c.send(u)
        results = parse_html(html)
        work_urls = (x for x in map(parse_url, results) if x)
        do_work(work_urls)

  do_work([start_url])
  c.close()


if __name__ == '__main__':
  c = consumer()
  produce(c)
  print(len(history_urls))