from scrapy import Request
from scrapy.spiders import Spider
from lianjia_home.items import LianjiaHomeItem
class HomeSpider(Spider):
 name = "home"
 current_page=1 #起始頁
 def start_requests(self): #初始請求
  url="https://su.lianjia.com/ershoufang/"
  yield Request(url=url)
 def parse(self, response): #解析函數
  list_selctor=response.xpath("http://li/div[@class='info clear']")
  for one_selector in list_selctor:
try:
 #房屋名稱
 name=one_selector.xpath("http://div[@class='flood']/div[@class='positionInfo']/a/text()").extract_first()
 #其他信息
 other=one_selector.xpath("http://div[@class='address']/div[@class='houseInfo']/text()").extract_first()
 other_list=other.split("|")
 type=other_list[0].strip(" ")#戶型
 area = other_list[1].strip(" ") #面積
 direction=other_list[2].strip(" ") #朝向
 fitment=other_list[3].strip(" ") #裝修
 price_list=one_selector.xpath("div[@class='priceInfo']//span/text()")
 # 總價
 total_price=price_list[0].extract()
 # 單價
 unit_price=price_list[1].extract()
 item=LianjiaHomeItem()
 item["name"]=name.strip(" ")
 item["type"]=type
 item["area"] = area
 item["direction"] = direction
 item["fitment"] = fitment
 item["total_price"] = total_price
 item["unit_price"] = unit_price
#生成詳細頁
 url = one_selector.xpath("div[@class='title']/a/@href").extract_first()
 yield Request(url=url,
  meta={"item":item}, #把item作為數據v傳遞
  callback=self.property_parse) #爬取詳細頁
except:
 print("error")
  #獲取下一頁
self.current_page+=1
if self.current_page<=100:
 next_url="https://su.lianjia.com/ershoufang/pg%d"%self.current_page
 yield Request(url=next_url)

 def property_parse(self,response):#詳細頁
  #配備電梯
  elevator=response.xpath("http://div[@class='base']/div[@class='content']/ul/li[last()]/text()").extract_first()
  item=response.meta["item"]
  item["elevator"]=elevator
  yield item

import re
from scrapy.exceptions import DropItem
class LianjiaHomePipeline:#數據的清洗
 def process_item(self, item, spider):
  #面積
  item["area"]=re.findall("\d+\.?\d*",item["area"])[0] #提取數字并存儲
  #單價
  item["unit_price"] = re.findall("\d+\.?\d*", item["unit_price"])[0] #提取數字并存儲
  #如果有不完全的數據，則拋棄
  if item["direction"] =="暫無數據":
raise DropItem("無數據，拋棄：%s"%item)
  return item
class CSVPipeline(object):
 file=None
 index=0 #csv文件行數判斷
 def open_spider(self,spider): #爬蟲開始前，打開csv文件
  self.file=open("home.csv","a",encoding="utf=8")
 def process_item(self, item, spider):#按要求存儲文件。
  if self.index ==0:
column_name="name,type,area,direction,fitment,elevator,total_price,unit_price\n"
self.file.write(column_name)#插入第一行的索引信息
self.index=1
  home_str=item["name"]+","+item["type"]+","+item["area"]+","+item["direction"]+","+item["fitment"]+","+item["elevator"]+","+item["total_price"]+","+item["unit_price"]+"\n"
  self.file.write(home_str) #插入獲取的信息
  return item
 def close_soider(self,spider):#爬蟲結束后關閉csv
  self.file.close()

USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36'
#為裝成瀏覽器
ROBOTSTXT_OBEY = False #不遵循robots協議
ITEM_PIPELINES = {
 'lianjia_home.pipelines.LianjiaHomePipeline': 300,
 #先進行數字提取
 'lianjia_home.pipelines.CSVPipeline': 400
 #在進行數據的儲存
 #執(zhí)行順序由后邊的數字決定
}

from scrapy import cmdline
cmdline.execute("scrapy crawl home" .split())