python爬虫入门八：多进程/多线程

import threading

print(threading.active_count()) # 显示当前激活的线程数
print(threading.enumerate()) # 显示当前激活的线程
print(threading.current_thread()) # 当前运行的线程 


def thread_job():
    print('This is a thread of %s' % threading.current_thread())

def main():
    thread = threading.Thread(target=thread_job,) # 添加一个线程
    thread.start() # 开始该线程

if __name__ == '__main__':
    main()

import threading
import time

# 定义一个fun，传入线程
def T1_job():
    print('T1 start\n')
    for i in range(10):
        time.sleep(0.1)
    print('T1 finish\n')
        
def T2_job():
    print('T2 start\n')
    print('T2 finish\n')
        
def main():
    thread1 = threading.Thread(target=T1_job, name='T1') # 添加线程，准备执行thread_job，命名T1
    thread2 = threading.Thread(target=T2_job, name='T2')
    
    thread1.start() # 执行该线程,没有添加join的时候，同步执行main和thread_job
    thread2.start()
    
    thread1.join() # 等待thread1完成后才进行下一步-主程序
    thread2.join() # 等待thread2完成后才进行下一步-主程序
    print('all done')

if __name__ == '__main__':
    main()

import threading
from queue import Queue

def job(l, q):
    for i in range(len(l)):
        l[i] = l[i]**2
    q.put(l) # 线程中，return获取的值无法提取，需要放入q中

def multithreading():
    q = Queue() # 队列
    threads = [] # 全部线程
    data = [[1, 2, 3], [3, 4, 5], [4,4,4], [5,5,5]]
    for i in range(4):
        # 4个线程来执行job函数
        t = threading.Thread(target=job, args=(data[i], q))
        t.start()
        threads.append(t) # 当前线程加入全部线程中
        
    # 对主线程中的每一个线程都执行join()
    for thread in threads:
        thread.join()
   
    results = [] # 保存结果
    for _ in range(4):
        results.append(q.get()) # 从q中拿出值，每次只能按顺序拿出一个值
    print(results)
    
if __name__ == '__main__':
    multithreading()

# [[1, 4, 9], [9, 16, 25], [16, 16, 16], [25, 25, 25]]

# lock锁，当前线程运行完成后才进行下一进程
import threading

def job1():
    global A, lock
    lock.acquire() # 打开锁
    for i in range(10):
        A += 1
        time.sleep(0.2)
        print('job1', A)
    lock.release() # 关闭锁
    
def job2():
    global A, lock
    lock.acquire() # 打开锁
    for i in range(10):
        A += 10
        time.sleep(0.2)
        print('job2', A)
    lock.release() # 关闭锁
    
if __name__ == '__main__':
    lock = threading.Lock() # lock锁
    A = 0
    t1 = threading.Thread(target=job1)
    t2 = threading.Thread(target=job2)
    t1.start()
    t2.start()

from multiprocessing.dummy import Pool as ThreadPool # 线程池
import threading

def job(i):
    print(i, '\n', threading.current_thread())
    
if __name__ == '__main__':
    pool = ThreadPool(4) # 创建一个包含4个线程的线程池
    pool.map(job, range(12))
    pool.close() # 关闭线程池的写入
    pool.join() # 阻塞，保证子线程运行完毕后再继续主进程 

import multiprocessing as mp

def job(a,d):
    print('aaaaa')

if __name__=='__main__':
    p1 = mp.Process(target=job,args=(1,2)) # 添加一个进程
    p1.start()
    p1.join()

import multiprocessing as mp

def job(q):
    res=0
    for i in range(1000):
        res+=i+i**2+i**3
    q.put(res)    # 将值放入队列

if __name__=='__main__':
    q = mp.Queue() # Queue队列
    p1 = mp.Process(target=job,args=(q,))
    p2 = mp.Process(target=job,args=(q,))
    p1.start()
    p2.start()
    p1.join()
    p2.join()
    res1 = q.get() # 从队列中取出值
    res2 = q.get() # 从队列中取出值
    print(res1, res2)

import multiprocessing as mp

def job(x):
    return x*x

def multicore():
    pool = mp.Pool() # 定义一个进程池
    res = pool.map(job, range(100))
    print(res)

if __name__=='__main__':
    multicore()

import multiprocessing as mp

value1 = mp.Value('i', 0) # value接受单个数值，i表示一个带符号的整型
array = mp.Array('i', [1, 2, 3, 4]) # Array接受一个一维数组

array2 = mp.Array('i', [[1,2], [2,3]]) # 传入一个二维数组，错误，传入参数非一维数组

import multiprocessing as mp

def job(v, num, l):
    l.acquire() # 锁住
    for _ in range(5):
        time.sleep(0.1) 
        v.value += num # 获取共享内存
        print(v.value)
    l.release() # 释放

def multicore():
    l = mp.Lock() # 定义一个进程锁
    v = mp.Value('i', 0) # 定义共享内存
    p1 = mp.Process(target=job, args=(v,1,l)) # 需要将lock传入
    p2 = mp.Process(target=job, args=(v,3,l)) 
    p1.start()
    p2.start()
    p1.join()
    p2.join()

if __name__ == '__main__':
    multicore()