ource activate crawlenv
python initial.py file  # 初始化file模式
python crawl_user.py file 1 100  # file模式，1 100是開始、結束bilibili的uid

git clone https://github.com/cgDeepLearn/BilibiliCrawler.git

conda create -n crawlenv python=3.6
source activate crawlenv  # 激活虛擬環(huán)境
pip install -r requirements.txt

virtualenv crawlenv
source crawlenv/bin/activate  # 激活虛擬環(huán)境，windows下不用source
pip install -r requirements.txt  # 安裝項目依賴

[db_mysql]
user = test
password = test
host = localhost
port = 3306
dbname = testdb

[db_postgresql]
user = test
password = test
host = localhost
port = 5432
dbname = testdb

def get_db_args():
    """
    獲取數(shù)據(jù)庫配置信息
    """
    return dict(CONFIG.items('db_postgresql'))  # 如果安裝的是mysql,請將參數(shù)替換為db_mysql

# connect_str = "postgresql+psycopg2://{}:{}@{}:{}/{}".format(kwargs['user'], kwargs['password'], kwargs['host'], kwargs['port'], kwargs['dbname'])
# 若使用的是mysql，請將上面的connect_str替換成下面的
connect_str = "mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8".format(kwargs['user'], kwargs['password'], kwargs['host'], kwargs['port'], kwargs['dbname'])
# sqlite3，mongo等請移步我的DB_ORM項目，其他一些數(shù)據(jù)庫也將添加支持

python initial.py db # db模式，file模式請將db換成file
# file模式會將抓取結果保存在data目錄
# db模式會將數(shù)據(jù)保存在設置好的數(shù)據(jù)庫中
# 若再次以db模式運行將會drop所有表后再create，初次運行后請慎重再次使用!!!
# 如果修改添加了表，并不想清空數(shù)據(jù)，請運行 python create_all.py

python crawl_user.py db 1 10000 # crawl_user 抓取用戶數(shù)據(jù)，db 保存在數(shù)據(jù)庫中， 1 10000為抓取起止id
python crawl_video_ajax.py db 1 100 # crawl_video_ajax 抓取視頻ajax信息保存到數(shù)據(jù)庫中,
python crawl_user_video.py db 1 10000 #同時抓取user 和videoinfo
# 示例為uid從1到100的user如果有投稿視頻則抓取其投稿視頻的信息，
# 若想通過視頻id逐個抓取請運行python crawl_video_by_aid.py db 1 1000

nohup python crawl_user db 1 10000

nohup python crawl_video_ajax.py db 1 1000 > video_ajaxup_1_1000.out  # 輸出將保存在video_ajaxup_1_1000.out中

produce 1_1
consumed 1_1
...

# utils/pcModels.py
print('[+] produce %s_%s' % (index, pitem))  # 請注釋掉

print('[-] consumed %s_%s\n' % (index, data))  # 請注釋掉