基于機(jī)器學(xué)習(xí)的PM5預(yù)測(cè)與分析系統(tǒng)代碼

1、基于深度學(xué)習(xí)pm2.5預(yù)警分析系統(tǒng)_init_Spiders文件# ThispackagewillcontainthespidersofyourScrapyproject# Pleaserefertothedocumentationforinformationonhowtocreateandmanage# yourspiders.Items# -*-coding:utf-8-*-# Defineherethemodelsforyourscrapeditems# Seedocumentationin:# /en/latest/topics/items.ht

2、mlimportscrapyclassHistoryCityItem(scrapy.Item):city_name=scrapy.Field()city_url=scrapy.Field()classHistoryMonthItem(scrapy.Item):city_id=scrapy.Field()city_name=scrapy.Field()hm_year=scrapy.Field()hm_month=scrapy.Field()hm_aqi=scrapy.Field()hm_aqi_min=scrapy.Field()hm_aqi_max=scrapy.Field()hm_quali

3、ty=scrapy.Field()hm_pm25=scrapy.Field()hm_pm10=scrapy.Field()hm_so2=scrapy.Field()hm_co=scrapy.Field()hm_no2=scrapy.Field()hm_o3=scrapy.Field()hm_rank=scrapy.Field()hm_day_url=scrapy.Field()classHistoryDayItem(scrapy.Item):city_id=scrapy.Field()city_name=scrapy.Field()hd_date=scrapy.Field()hd_aqi=sc

4、rapy.Field()hd_aqi_min=scrapy.Field()hd_aqi_max=scrapy.Field()hd_quality=scrapy.Field()hd_pm25=scrapy.Field()hd_pm10=scrapy.Field()hd_so2=scrapy.Field()hd_co=scrapy.Field()hd_no2=scrapy.Field()hd_o3=scrapy.Field()hd_rank=scrapy.Field()classWeatherCityItem(scrapy.Item):city_name=scrapy.Field()city_ur

5、l=scrapy.Field()classWeatherDayItem(scrapy.Item):weather_date=scrapy.Field()city_id=scrapy.Field()city_name=scrapy.Field()weather_am=scrapy.Field()weather_pm=scrapy.Field()weather_top=scrapy.Field()weather_down=scrapy.Field()weather_am_wind_type=scrapy.Field()weather_am_wind_level=scrapy.Field()weat

6、her_pm_wind_type=scrapy.Field()weather_pm_wind_level=scrapy.Field()middlewares#-*-coding:utf-8-*-# Defineherethemodelsforyourspidermiddleware# Seedocumentationin:# /en/latest/topics/spider-middleware.htmlfromscrapyimportsignalsimportchardetclassSpiderSpiderMiddleware(object):# No

7、tallmethodsneedtobedefined.Ifamethodisnotdefined,# scrapyactsasifthespidermiddlewaredoesnotmodifythe# passedobjects.classmethoddeffrom_crawler(cls,crawler):# ThismethodisusedbyScrapytocreateyourspiders.s=cls()crawler.signals.connect(s.spider_opened,signal=signals.spider_opened)returnsdefprocess_spid

8、er_input(response,spider):# Calledforeachresponsethatgoesthroughthespider# middlewareandintothespider.# ShouldreturnNoneorraiseanexception.returnNonedefprocess_spider_output(response,result,spider):# CalledwiththeresultsreturnedfromtheSpider,after# ithasprocessedtheresponse.# MustreturnaniterableofR

9、equest,dictorItemobjects.foriinresult:yieldidefprocess_spider_exception(response,exception,spider):# Calledwhenaspiderorprocess_spider_input()method# (fromotherspidermiddleware)raisesanexception.# ShouldreturneitherNoneoraniterableofResponse,dict#orItemobjects.passdefprocess_start_requests(start_req

10、uests,spider):# Calledwiththestartrequestsofthespider,andworks# similarlytotheprocess_spider_output()method,except# thatitdoesn'thavearesponseassociated.# Mustreturnonlyrequests(notitems).forrinstart_requests:yieldrdefspider_opened(self,spider):('Spideropened:%s'%spider

11、.name)classSpiderTianQiHouBaoMiddleware(object):defprocess_response(self,request,response,spider):# encoding=chardet.detect(response.body)# ifencoding'encoding'='GB2312':# printresponse.headers._dict_.keys()#Content-Typeresponse.headers'content-type'='text/html;charset=gb

12、2312'returnresponsemodelfromsqlalchemyimportColumn,Date,DateTime,Integer,String,Text,Float,textfromsqlalchemy.ext.declarativeimportdeclarative_basedefto_dict(self)::getattr(self,,None)forcinself._table_.columnsBase=declarative_base()metadata=Base.metadataBase.to_dict=to_dictcla

13、ssHistoryCity(Base):_tablename_='history_city'city_id=Column(Integer,primary_key=True)city_name=Column(String(30),server_default=text("'':charactervarying")city_url=Column(String(1024),server_default=text("'':charactervarying")classHistoryMonth(Base):_tabl

14、ename_='history_month'hm_id=Column(Integer,primary_key=True)city_id=Column(Integer)city_name=Column(String(30),server_default=text("'':charactervarying")hm_year=Column(Integer)hm_month=Column(Integer)hm_aqi=Column(Integer)hm_aqi_min=Column(Integer)hm_aqi_max=Column(Integer)

15、hm_quality=Column(String(30),server_default=text("'':charactervarying")hm_pm25=Column(Float)hm_pm10=Column(Float)hm_so2=Column(Float)hm_co=Column(Float)hm_no2=Column(Float)hm_o3=Column(Float)hm_rank=Column(Integer)hm_day_url=Column(String(255),server_default=text("'':c

16、haractervarying")classHistoryDay(Base):_tablename_='history_day'hd_id=Column(Integer,primary_key=True)city_id=Column(Integer)city_name=Column(String(30),server_default=text("'':charactervarying")hd_date=Column(Date)hd_aqi=Column(Integer)hd_aqi_min=Column(Integer)hd_aqi

17、_max=Column(Integer)hd_quality=Column(String(30),server_default=text("'':charactervarying")hd_pm25=Column(Float)hd_pm10=Column(Float)hd_so2=Column(Float)hd_co=Column(Float)hd_no2=Column(Float)hd_o3=Column(Float)hd_rank=Column(Integer)classWeatherCity(Base):_tablename_='weather_

18、city'city_id=Column(Integer,primary_key=True)city_name=Column(String(30),server_default=text("'':charactervarying")city_url=Column(String(1024),server_default=text("'':charactervarying")classWeatherDay(Base):_tablename_='weather_day'weather_id=Column(I

19、nteger,primary_key=True)weather_date=Column(Date)city_id=Column(Integer)city_name=Column(String(30),server_default=text("'':charactervarying")weather_am=Column(String(30),server_default=text("'':charactervarying")weather_pm=Column(String(30),server_default=text(&q

20、uot;'':charactervarying")weather_top=Column(Integer)weather_down=Column(Integer)weather_am_wind_typevarying")weather_am_wind_levelvarying")weather_pm_wind_typevarying")weather_pm_wind_levelColumn(String(30),=Column(String(30),=Column(String(30),=Column(String(30),server_d

21、efault=text("'':characterserver_default=text("'':characterserver_default=text("'':characterserver_default=text("'':charactervarying")pipelines# -*-coding:utf-8-*-# Defineyouritempipelineshere# Don'tforgettoaddyourpipelinetotheITEM_PIPELINE

22、Ssetting# See:/en/latest/topics/item-pipeline.htmlimportjsonfromscrapy.exceptionsimportDropItemfromsqlalchemyimportcreate_enginefromsqlalchemy.ormimportsessionmakerfromitemsimportHistoryCityItem,HistoryMonthItem,HistoryDayItemfrommodelimportHistoryCity,HistoryDay,HistoryMonthfrom

23、itemsimportWeatherCityItem,WeatherDayItemfrommodelimportWeatherCity,WeatherDayclassHistoryPipeline(object):def_init_(self,sqlalchemy_database_uri,sqlalchemy_pool_size):self.engine=create_engine(sqlalchemy_database_uri,pool_size=sqlalchemy_pool_size)self.db_session=sessionmaker(autocommit=False,autof

24、lush=False,bind=self.engine)classmethoddeffrom_crawler(cls,crawler):returncls(sqlalchemy_database_uri=crawler.settings.get('SQLALCHEMY_DATABASE_URI_MYSQL'),sqlalchemy_pool_size=crawler.settings.get('SQLALCHEMY_POOL_SIZE',8),)defopen_spider(self,spider):self.session=self.db_session()d

25、efprocess_item(self,item,spider):try:ifisinstance(item,HistoryCityItem):#數(shù)據(jù)入庫(kù)service_item=HistoryCity(*item)row_sql="selectcount(*)cntfromhistory_citywherecity_name=:city_name"row_data="city_name":service_item.city_namerow=self.session.execute(row_sql,row_data).fetchone()ifnotrow

26、'cnt':self.session.add(service_item)mit()elifisinstance(item,HistoryMonthItem):service_item=HistoryMonth(*item)row_sql="select*fromhistory_citywherecity_name=:city_namerow_data="city_name":service_item.city_namerow=self.session.execute(row_sql,row_data).fetchone()if'city_i

27、d'inrow:service_item.city_id=row'city_id'#printu"HistoryMonthItem:%s"json.dumps(dict(service_item),indent=4,ensure_ascii=False)row_sql="selectcount(*)cntfromhistory_month""wherecity_name=:city_nameand=:hm_yearandhm_month=:hm_month"row_data="city_name&qu

28、ot;:service_item.city_name,"hm_year":service_item.hm_year,"hm_month":service_item.hm_monthrow=self.session.execute(row_sql,row_data).fetchone()ifnotrow'cnt':self.session.add(service_item)mit()elifisinstance(item,HistoryDayItem):service_item=HistoryDay(*item)row_sql="

29、select*fromhistory_citywhere=:city_name"row_data="city_name":service_item.city_namerow=self.session.execute(row_sql,row_data).fetchone()if'city_id'inrow:service_item.city_id=row'city_id'#printu"HistoryMonthItem:%s"json.dumps(dict(service_item),indent=4,ensure

30、_ascii=False)row_sql="selectcount(*)cntfromhistory_day""wherecity_name=:city_nameand=:hd_date"row_data="city_name":service_item.city_name,"hd_date":service_item.hd_date,row=self.session.execute(row_sql,row_data).fetchone()ifnotrow'cnt':self.session.add

31、(service_item)mit()exceptExceptionase:self.session.rollback()hm_yearcity_namehd_dateraiseereturnitemdefclose_spider(self,spider):self.session.close()classWeatherPipeline(object):def_init_(self,sqlalchemy_database_uri,sqlalchemy_pool_size):self.engine=create_engine(sqlalchemy_database_uri,pool_size=s

32、qlalchemy_pool_size)self.db_session=sessionmaker(autocommit=False,autoflush=False,bind=self.engine)classmethoddeffrom_crawler(cls,crawler):returncls(sqlalchemy_database_uri=crawler.settings.get('SQLALCHEMY_DATABASE_URI_MYSQL'),sqlalchemy_pool_size=crawler.settings.get('SQLALCHEMY_POOL_SIZE',8),)defopen_spider(self,spider):self.session=self.db_session()defprocess_item(self,item,spider):try:ifisinstance(item,WeatherCityItem):#數(shù)據(jù)入庫(kù)service_item=WeatherCi